Patch/ngc docs

.pre-commit-config.yaml

@@ -10,6 +10,7 @@ repos:
     rev: v4.4.0
     hooks:
       - id: check-added-large-files
+        args: ["--maxkb=500"]
       - id: check-ast
       - id: check-case-conflict
       - id: check-docstring-first
@@ -24,8 +25,8 @@ repos:
       - id: end-of-file-fixer
       - id: fix-byte-order-marker
       - id: mixed-line-ending
+        args: ["--fix=lf"]
       - id: requirements-txt-fixer
-      - id: trailing-whitespace
 
   # Ruff should be executed before other formatters.
   - repo: https://github.com/astral-sh/ruff-pre-commit
@@ -45,25 +46,16 @@ repos:
     hooks:
       - id: shellcheck
 
-  - repo: https://github.com/google/pyink
-    rev: 23.5.0
-    hooks: # Using PyInk, the Google fork of Black, for Python code formatting.
-      - id: pyink # Update to the `ruff` auto-formatter later.
-
-  - repo: https://github.com/PyCQA/prospector
-    rev: v1.9.0
-    hooks:
-      - id: prospector
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks: # Update to the `ruff` auto-formatter later.
+      - id: black
 
   - repo: https://github.com/nbQA-dev/nbQA
     rev: 1.7.0
     hooks:
       - id: nbqa-ruff # Run `ruff` on Jupyter Notebooks.
-
-  - repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks: # Using black for Jupyter Notebook formatting.
-      - id: black-jupyter
+      - id: nbqa-black # Remove later when the `ruff` auto-formatter is ready.
 
   - repo: local # Clear all Jupyter outputs before committing.
     hooks: # Comment this out if it is not the desired behavior.

Makefile

@@ -121,7 +121,7 @@ ls:  # List all services.
 
 # Utility for installing Docker Compose on Linux (but not WSL) systems.
 # Visit https://docs.docker.com/compose/install for the full documentation.
-COMPOSE_VERSION = v2.17.2
+COMPOSE_VERSION = v2.18.1
 COMPOSE_OS_ARCH = linux-x86_64
 COMPOSE_URL = https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-${COMPOSE_OS_ARCH}
 COMPOSE_PATH = ${HOME}/.docker/cli-plugins

dockerfiles/ngc.Dockerfile

@@ -19,17 +19,18 @@ RUN git clone --depth 1 ${PURE_URL} /opt/zsh/pure
 RUN git clone --depth 1 ${ZSHA_URL} /opt/zsh/zsh-autosuggestions
 RUN git clone --depth 1 ${ZSHS_URL} /opt/zsh/zsh-syntax-highlighting
 
-# Copy and install `apt` requirements for ngc images.
+# Copy `apt` and `conda` requirements for ngc images.
 COPY --link ../reqs/ngc-apt.requirements.txt /tmp/apt/requirements.txt
 COPY --link ../reqs/ngc-conda.requirements.txt /tmp/req/requirements.txt
 
 ########################################################################
 FROM ${BASE_IMAGE} AS install-conda
 # Starting with the 22.11 PyTorch NGC container, miniforge is removed
 # and all Python packages are installed in the default Python environment.
-# A separate conda installation is provided to allow conda installation.
-# Also, this will prevent user-installed Python packages from overwriting
-# the default packages in the NGC image, which have been carefully configured.
+# A separate conda installation is provided to allow conda installation,
+# which will also prevent user-installed Python packages from overwriting
+# those in the NGC image, which have been carefully configured.
+# NGC images prior to 22.11 are **incompatible** with the current Dockerfile.
 
 LABEL [email protected]
 ENV LANG=C.UTF-8
@@ -60,7 +61,7 @@ RUN --mount=type=cache,target=${PIP_CACHE_DIR},sharing=locked \
     $conda install --copy -p /opt/conda --file /tmp/req/requirements.txt && \
     printf "channels:\n  - conda-forge\n  - nodefaults\n" > /opt/conda/.condarc
 
-# Create a symbolic link to add `site-packages` to the PYTHONPATH later.
+# Create a symbolic link to add Python `site-packages` to `PYTHONPATH` later on.
 RUN ln -s \
     /opt/conda/lib/$(python -V | awk -F '[ \.]' '{print "python" $2 "." $3}') \
     /opt/conda/lib/python3
@@ -76,7 +77,7 @@ ARG PYTHONDONTWRITEBYTECODE=1
 ARG PYTHONUNBUFFERED=1
 
 # The base NGC image sets `SHELL=bash`. Docker cannot unset an `ENV` variable,
-# ergo, `SHELL=''` is used for best compatibility with the other services.
+# therefore, `SHELL=''` is used for best compatibility with the other services.
 ENV SHELL=''
 
 # Install `apt` requirements.
@@ -146,6 +147,7 @@ FROM train-interactive-${INTERACTIVE_MODE} AS train
 ENV KMP_BLOCKTIME=0
 ENV KMP_AFFINITY="granularity=fine,compact,1,0"
 # Use `/opt/conda/lib/libiomp5.so` for older NGC images using `conda`.
+# Using the older system MKL to prevent version clashes with NGC packages.
 ENV LD_PRELOAD=/usr/local/lib/libiomp5.so:${LD_PRELOAD}
 
 # Enable Intel MKL optimizations on AMD CPUs.
@@ -155,9 +157,7 @@ ENV MKL_DEBUG_CPU_TYPE=5
 RUN echo 'int mkl_serv_intel_cpu_true() {return 1;}' > /tmp/fakeintel.c && \
     gcc -shared -fPIC -o /usr/local/bin/libfakeintel.so /tmp/fakeintel.c
 ENV LD_PRELOAD=/usr/local/bin/libfakeintel.so:${LD_PRELOAD}
-
-# The `jemalloc` binary location and name may change between NGC versions.
-# The `x86_64` architecture is also hard-coded by necessity.
+# Configure Jemalloc as the default memory allocator.
 ENV LD_PRELOAD=/opt/conda/lib/libjemalloc.so:${LD_PRELOAD}
 ENV MALLOC_CONF="background_thread:true,metadata_thp:auto,dirty_decay_ms:30000,muzzy_decay_ms:30000"
 
@@ -167,7 +167,7 @@ RUN chmod 711 /root
 ARG PROJECT_ROOT=/opt/project
 ENV PATH=${PROJECT_ROOT}:${PATH}
 # Search for additional Python packages installed via `conda`.
-# This requires `/opt/conda/lib/python3` to be created as a symlink.
+# This requires `/opt/conda/lib/python3` to be created as a symlink beforehand.
 ENV PYTHONPATH=${PROJECT_ROOT}:/opt/conda/lib/python3/site-packages
 WORKDIR ${PROJECT_ROOT}
 CMD ["/bin/zsh"]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.black]
 line-length = 100
-target-version = ['py38', 'py39', 'py310']
+target-version = ['py310', 'py311']
 include = '\.pyi?$'
 # Black excludes files in `.gitignore` if no `exclude` is specified.
 # https://black.readthedocs.io/en/stable/usage_and_configuration/file_collection_and_discovery.html#gitignore
@@ -9,7 +9,7 @@ include = '\.pyi?$'
 # PyInk is the Google fork of Black.
 # https://github.com/google/pyink
 line-length = 100
-target-version = ['py38', 'py39', 'py310']
+target-version = ['py310', 'py311']
 include = '\.pyi?$'
 
 [tool.pytest.ini_options]
@@ -28,7 +28,7 @@ testpaths = [
 
 [tool.ruff]
 line-length = 100
-target-version = 'py38'
+target-version = 'py310'
 select = [
 #    "A", # flake8-builtins
 #    "ARG", # flake8-unused-arguments