NVIDIA
diff --git a/‎.dockerignore
+2 b/‎.dockerignore
+2
diff --git a/‎.github/PULL_REQUEST_TEMPLATE.md
+3 b/‎.github/PULL_REQUEST_TEMPLATE.md
+3
diff --git a/‎.github/labeler.yml
+8 b/‎.github/labeler.yml
+8
diff --git a/‎.pre-commit-config.yaml
+1-1 b/‎.pre-commit-config.yaml
+1-1
diff --git a/‎.readthedocs.yml
+5-1 b/‎.readthedocs.yml
+5-1
diff --git a/‎Dockerfile
+65-20 b/‎Dockerfile
+65-20
@@ -17,3 +17,5 @@ coverage.xml
 .git
 **/*.nemo
 **/*.ckpt
+workspace
+nemo_experiments
@@ -14,6 +14,9 @@ Add a one line overview of what this PR aims to accomplish.
 # Add a code snippet demonstrating how to use this 
 ```
 
+# Jenkins CI
+To run Jenkins, a NeMo User with write access must comment `jenkins` on the PR.
+
 # Before your PR is "Ready for review"
 **Pre checks**:
 - [ ] Make sure you read and followed [Contributor guidelines](https://github.com/NVIDIA/NeMo/blob/main/CONTRIBUTING.md)
 
@@ -3,25 +3,33 @@ ASR:
 - examples/asr/**/*
 - tutorials/asr/**/*
 - docs/source/asr/**/*
+- tests/collections/asr/**
 
 NLP:
 - nemo/collections/nlp/**/*
 - examples/nlp/**/*
 - tutorials/nlp/**/*
 - docs/source/nlp/**/*
+- tests/collections/nlp/**
 
 Speaker Tasks:
 - examples/speaker_tasks/**/*
 - tutorials/speaker_tasks/**/*
 
 TTS:
 - nemo/collections/tts/**/*
+- nemo/collections/common/tokenizers/text_to_speech/**
 - examples/tts/**/*
 - tutorials/tts/**/*
 - docs/source/tts/**/*
+- scripts/dataset_processing/tts/**
+- scripts/tts_dataset_files/**
+- tests/collections/tts/**
+- tests/collections/common/tokenizers/text_to_speech/**
 
 core:
 - nemo/core/**/*
+- tests/core/**
 
 common:
 - nemo/collections/common/**/*
 
@@ -28,7 +28,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
       - id: check-added-large-files
-        args: ['--maxkb=1000']
+        args: ['--maxkb=5000']
       - id: requirements-txt-fixer
 
   - repo: https://github.com/PyCQA/isort
 
@@ -20,12 +20,16 @@
 # Required field.
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
 # Build documentation in the docs/ directory with Sphinx.
 sphinx:
   configuration: docs/source/conf.py
 
 # Set the version of Python and requirements required to build your docs
 python:
-  version: 3.8
   install:
     - requirements: requirements/requirements_docs.txt
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:23.06-py3
+ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:23.12-py3
 
 # build an image that includes only the nemo dependencies, ensures that dependencies
 # are included first for optimal caching, and useful for building a development
@@ -31,7 +31,7 @@ ARG REQUIRE_AIS_CLI=false
 
 # Ensure apt-get won't prompt for selecting options
 ENV DEBIAN_FRONTEND=noninteractive
-# libavdevice-dev rerquired for latest torchaudio
+# libavdevice-dev required for latest torchaudio
 RUN apt-get update && \
   apt-get upgrade -y && \
   apt-get install -y \
@@ -42,15 +42,48 @@ RUN apt-get update && \
   libavdevice-dev && \
   rm -rf /var/lib/apt/lists/*
 
-WORKDIR /workspace/
+# libtool, ... , libgts-dev are required for graphviz
+# graphviz is required for k2 and pynini visualization
+RUN apt-get update && \
+  apt-get install -y \
+  libtool \
+  libltdl-dev \
+  automake \
+  autoconf \
+  bison \
+  flex \
+  tcl \
+  ghostscript \
+  libgd-dev \
+  fontconfig \
+  libcairo2-dev \
+  libpango1.0-dev \
+  libgts-dev && \
+  rm -rf /var/lib/apt/lists/*
 
-WORKDIR /tmp/
-# TODO: Remove once this Apex commit (5/12/23) is included in PyTorch
-# container
+WORKDIR /workspace/
+# Install megatron core, this can be removed once 0.3 pip package is released
+# We leave it here in case we need to work off of a specific commit in main
+RUN git clone https://github.com/NVIDIA/Megatron-LM.git && \
+  cd Megatron-LM && \
+  git checkout 27cbe46714a50c43ed290f1b1472db8d2780c55c && \
+  pip install .
+
+# Performance optimizations for distributed optimizer: https://github.com/NVIDIA/apex/pull/1771
 RUN git clone https://github.com/NVIDIA/apex.git && \
   cd apex && \
-  git checkout 8b7a1ff183741dd8f9b87e7bafd04cfde99cea28 && \
-  pip3 install -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" --global-option="--fast_layer_norm" --global-option="--distributed_adam" --global-option="--deprecated_fused_adam" ./
+  git checkout b496d85fb88a801d8e680872a12822de310951fd && \
+  pip install -v --no-build-isolation --disable-pip-version-check --no-cache-dir --config-settings "--build-option=--cpp_ext --cuda_ext --fast_layer_norm --distributed_adam --deprecated_fused_adam" ./
+
+# Transformer Engine 1.2.0
+RUN git clone https://github.com/NVIDIA/TransformerEngine.git && \
+  cd TransformerEngine && \
+  git fetch origin 4f9662fbe621671f5f905e772fc1138953af77f6 && \
+  git checkout FETCH_HEAD && \
+  git submodule init && git submodule update && \
+  NVTE_FRAMEWORK=pytorch NVTE_WITH_USERBUFFERS=1 MPI_HOME=/usr/local/mpi pip install .
+
+WORKDIR /tmp/
 
 # uninstall stuff from base container
 RUN pip3 uninstall -y sacrebleu torchtext
@@ -67,19 +100,20 @@ RUN INSTALL_MSG=$(/bin/bash /tmp/torchaudio_build/scripts/installers/install_tor
   else echo "Skipping failed torchaudio installation"; fi \
   else echo "torchaudio installed successfully"; fi
 
-# install nemo dependencies
-WORKDIR /tmp/nemo
-COPY requirements .
-RUN for f in $(ls requirements*.txt); do pip3 install --disable-pip-version-check --no-cache-dir -r $f; done
-
-# install flash attention dependencies
-RUN pip install flash-attn
-# pinned triton version for flash-attention https://github.com/HazyResearch/flash-attention/blob/main/flash_attn/flash_attn_triton.py#L3
-RUN pip install triton==2.0.0.dev20221202
+COPY scripts /tmp/nemo/scripts/
+# install correct graphviz version (k2 and pynini visualization tool), skip if installation fails
+RUN INSTALL_MSG=$(/bin/bash /tmp/nemo/scripts/installers/install_graphviz.sh --docker); INSTALL_CODE=$?; \
+  echo ${INSTALL_MSG}; \
+  if [ ${INSTALL_CODE} -ne 0 ]; then \
+  echo "graphviz installation failed";  \
+  if [ "${REQUIRE_K2}" = true ]; then \
+  exit ${INSTALL_CODE};  \
+  else echo "Skipping failed graphviz installation"; fi \
+  else echo "graphviz installed successfully"; fi
 
 # install k2, skip if installation fails
 COPY scripts /tmp/nemo/scripts/
-RUN INSTALL_MSG=$(/bin/bash /tmp/nemo/scripts/speech_recognition/k2/setup.sh); INSTALL_CODE=$?; \
+RUN INSTALL_MSG=$(/bin/bash /tmp/nemo/scripts/installers/install_k2.sh); INSTALL_CODE=$?; \
   echo ${INSTALL_MSG}; \
   if [ ${INSTALL_CODE} -ne 0 ]; then \
   echo "k2 installation failed";  \
@@ -88,13 +122,24 @@ RUN INSTALL_MSG=$(/bin/bash /tmp/nemo/scripts/speech_recognition/k2/setup.sh); I
   else echo "Skipping failed k2 installation"; fi \
   else echo "k2 installed successfully"; fi
 
+# install nemo dependencies
+WORKDIR /tmp/nemo
+ENV LHOTSE_REQUIRE_TORCHAUDIO=0
+COPY requirements .
+RUN for f in $(ls requirements*.txt); do pip3 install --disable-pip-version-check --no-cache-dir -r $f; done
+
+# install flash attention
+RUN pip install flash-attn
+# install numba for latest containers
+RUN pip install numba>=0.57.1
+
 # copy nemo source into a scratch image
 FROM scratch as nemo-src
 COPY . .
 
 # start building the final container
 FROM nemo-deps as nemo
-ARG NEMO_VERSION=1.20.0
+ARG NEMO_VERSION=1.23.0
 
 # Check that NEMO_VERSION is set. Build will fail without this. Expose NEMO and base container
 # version information as runtime environment variable for introspection purposes
@@ -103,7 +148,7 @@ RUN /usr/bin/test -n "$NEMO_VERSION" && \
   /bin/echo "export BASE_IMAGE=${BASE_IMAGE}" >> /root/.bashrc
 
 # Install NeMo
-RUN --mount=from=nemo-src,target=/tmp/nemo cd /tmp/nemo && pip install ".[all]"
+RUN --mount=from=nemo-src,target=/tmp/nemo,rw cd /tmp/nemo && pip install ".[all]"
 
 # Check install
 RUN python -c "import nemo.collections.nlp as nemo_nlp" && \