Low touch upgrade to TensorFlow 2.3

Dockerfile.train.tmpl

@@ -1,6 +1,6 @@
 # Please refer to the TRAINING documentation, "Basic Dockerfile for training"
 
-FROM tensorflow/tensorflow:1.15.4-gpu-py3
+FROM tensorflow/tensorflow:2.3.1-gpu
 ENV DEBIAN_FRONTEND=noninteractive
 
 ENV DEEPSPEECH_REPO=#DEEPSPEECH_REPO#

doc/TRAINING.rst

@@ -10,7 +10,7 @@ Prerequisites for training a model
 
 * `Python 3.6 <https://www.python.org/>`_
 * Mac or Linux environment
-* CUDA 10.0 / CuDNN v7.6 per `Dockerfile <https://hub.docker.com/layers/tensorflow/tensorflow/1.15.4-gpu-py3/images/sha256-a5255ae38bcce7c7610816c778244309f8b8d1576e2c0023c685c011392958d7?context=explore>`_.
+* CUDA 10.1 / CuDNN v7.6 per `Dockerfile <https://hub.docker.com/layers/tensorflow/tensorflow/2.3.1-gpu/images/sha256-1d0736e46ae9a961c2111394a43e0bfd266e6151a90d613b6f86229cf01e40e5?context=explore  >`_.
 
 Getting the training code
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -72,7 +72,7 @@ If you have a capable (NVIDIA, at least 8GB of VRAM) GPU, it is highly recommend
 .. code-block:: bash
 
    pip3 uninstall tensorflow
-   pip3 install 'tensorflow-gpu==1.15.4'
+   pip3 install 'tensorflow-gpu==2.3.1'
 
 Please ensure you have the required `CUDA dependency <https://www.tensorflow.org/install/source#gpu>`_ and/or :ref:`Prerequisites <cuda-training-deps>`.
 
@@ -83,7 +83,7 @@ It has been reported for some people failure at training:
    tensorflow.python.framework.errors_impl.UnknownError: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
         [[{{node tower_0/conv1d/Conv2D}}]]
 
-Setting the ``TF_FORCE_GPU_ALLOW_GROWTH`` environment variable to ``true`` seems to help in such cases. This could also be due to an incorrect version of libcudnn. Double check your versions with the :ref:`TensorFlow 1.15 documentation <cuda-training-deps>`.
+Setting the ``TF_FORCE_GPU_ALLOW_GROWTH`` environment variable to ``true`` seems to help in such cases. This could also be due to an incorrect version of libcudnn. Double check your versions with the :ref:`TensorFlow 2.3 documentation <cuda-training-deps>`.
 
 Basic Dockerfile for training
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

native_client/BUILD

@@ -169,33 +169,33 @@ cc_library(
             "//tensorflow/core/kernels:spectrogram_op",  # AudioSpectrogram
             "//tensorflow/core/kernels:bias_op",  # BiasAdd
             "//tensorflow/core/kernels:cast_op",  # Cast
-            "//tensorflow/core/kernels:concat_op",  # ConcatV2
             "//tensorflow/core/kernels:constant_op",  # Const, Placeholder
-            "//tensorflow/core/kernels:shape_ops",  # ExpandDims, Shape
-            "//tensorflow/core/kernels:gather_nd_op",  # GatherNd
+            "//tensorflow/core/kernels:shape_ops",  # ExpandDims
             "//tensorflow/core/kernels:identity_op",  # Identity
             "//tensorflow/core/kernels:immutable_constant_op",  # ImmutableConst (used in memmapped models)
-            "//tensorflow/core/kernels:deepspeech_cwise_ops",  # Less, Minimum, Mul
+            "//tensorflow/core/kernels:deepspeech_cwise_ops",  # Minimum, Mul, SelectV2, NotEqual
             "//tensorflow/core/kernels:matmul_op",  # MatMul
-            "//tensorflow/core/kernels:reduction_ops",  # Max
             "//tensorflow/core/kernels:mfcc_op",  # Mfcc
             "//tensorflow/core/kernels:no_op",  # NoOp
             "//tensorflow/core/kernels:pack_op",  # Pack
-            "//tensorflow/core/kernels:sequence_ops",  # Range
             "//tensorflow/core/kernels:relu_op",  # Relu
             "//tensorflow/core/kernels:reshape_op",  # Reshape
+            "//tensorflow/core/kernels:reduction_ops",  # Any
             "//tensorflow/core/kernels:softmax_op",  # Softmax
+            "//tensorflow/core/kernels:strided_slice_op",  # StridedSlice
+            "//tensorflow/core/kernels:slice_op",  # Slice
+            "//tensorflow/core/kernels:split_op",  # Split
             "//tensorflow/core/kernels:tile_ops",  # Tile
             "//tensorflow/core/kernels:transpose_op",  # Transpose
-            "//tensorflow/core/kernels:rnn_ops",  # BlockLSTM
+            "//tensorflow/core/kernels:unpack_op",  # Unpack
             # And we also need the op libs for these ops used in the model:
             "//tensorflow/core:audio_ops_op_lib",  # AudioSpectrogram, Mfcc
-            "//tensorflow/core:rnn_ops_op_lib",  # BlockLSTM
-            "//tensorflow/core:math_ops_op_lib",  # Cast, Less, Max, MatMul, Minimum, Range
-            "//tensorflow/core:array_ops_op_lib",  # ConcatV2, Const, ExpandDims, Fill, GatherNd, Identity, Pack, Placeholder, Reshape, Tile, Transpose
+            "//tensorflow/core:math_ops_op_lib",  # MatMul, Minimum
+            "//tensorflow/core:array_ops_op_lib",  # Const, ExpandDims, Identity, Pack, Placeholder, Reshape, Tile, Transpose
             "//tensorflow/core:no_op_op_lib",  # NoOp
             "//tensorflow/core:nn_ops_op_lib",  # Relu, Softmax, BiasAdd
             # And op libs for these ops brought in by dependencies of dependencies to silence unknown OpKernel warnings:
+            "//tensorflow/core:state_ops_op_lib",  # Variable, VariableV2, TemporaryVariable, DestroyTemporaryVariable, IsVariableInitialized
             "//tensorflow/core:dataset_ops_op_lib",  # UnwrapDatasetVariant, WrapDatasetVariant
             "//tensorflow/core:sendrecv_ops_op_lib",  # _HostRecv, _HostSend, _Recv, _Send
         ],

native_client/ctcdecode/path_trie.cpp

@@ -158,7 +158,7 @@ PathTrie* PathTrie::get_prev_word(std::vector<unsigned int>& output,
 
 void PathTrie::iterate_to_vec(std::vector<PathTrie*>& output) {
   // previous_timesteps might point to ancestors' timesteps
-  // therefore, children must be uptaded first
+  // therefore, children must be updated first
   for (auto child : children_) {
     child.second->iterate_to_vec(output);
   }

native_client/tflitemodelstate.cc

@@ -362,8 +362,8 @@ TFLiteModelState::infer(const vector<float>& mfcc,
 {
   const size_t num_classes = alphabet_.GetSize() + 1; // +1 for blank
 
-  // Feeding input_node
-  copy_vector_to_tensor(mfcc, input_node_idx_, n_frames*mfcc_feats_per_timestep_);
+  // Feeding input_node, zeroing remaining timesteps in case of a partial batch
+  copy_vector_to_tensor(mfcc, input_node_idx_, n_steps_*mfcc_feats_per_timestep_);
 
   // Feeding previous_state_c, previous_state_h
   assert(previous_state_c.size() == state_size_);

native_client/tfmodelstate.cc

@@ -213,14 +213,10 @@ TFModelState::infer(const std::vector<float>& mfcc,
   Tensor previous_state_c_t = tensor_from_vector(previous_state_c, TensorShape({BATCH_SIZE, (long long)state_size_}));
   Tensor previous_state_h_t = tensor_from_vector(previous_state_h, TensorShape({BATCH_SIZE, (long long)state_size_}));
 
-  Tensor input_lengths(DT_INT32, TensorShape({1}));
-  input_lengths.scalar<int>()() = n_frames;
-
   vector<Tensor> outputs;
   Status status = session_->Run(
     {
      {"input_node", input},
-     {"input_lengths", input_lengths},
      {"previous_state_c", previous_state_c_t},
      {"previous_state_h", previous_state_h_t}
     },

setup.py

@@ -67,14 +67,15 @@ def main():
         'llvmlite == 0.31.0', # for numba==0.47.0
         'librosa',
         'soundfile',
+        'tensorflow_addons >= 0.12.0',
     ]
 
     decoder_pypi_dep = [
         'ds_ctcdecoder == {}'.format(version)
     ]
 
     tensorflow_pypi_dep = [
-        'tensorflow == 1.15.4'
+        'tensorflow == 2.3.1'
     ]
 
     # Due to pip craziness environment variables are the only consistent way to

taskcluster/.build.yml

@@ -23,7 +23,7 @@ build:
   nc_asset_name: 'native_client.tar.xz'
   args:
     tests_cmdline: ''
-  tensorflow_git_desc: 'TensorFlow: v2.3.0-6-g23ad988'
+  tensorflow_git_desc: 'TensorFlow: v2.3.0-8-g4c4c6ac'
   test_model_task: ''
   homebrew:
     url: ''