Merge remote-tracking branch 'aws/master' into update-tf-2.3

* aws/master:
  [pytorch][release] Release pt1.6 Inference cpu, gpu and training cpu (aws#1074)
  [tensorflow, pytorch] [build] [test] [ec2, ecs, eks, sagemaker] Add EFA stack and tests (aws#1044)
  [pytorch][build][test] Update PT1.6.0 for pillow to 8.2.0 (aws#1071)
  Revert "[build,test] Disable dedicated telemetry tests and tags (aws#1045)" (aws#1055)

Author: jeet4320

pytorch/inference/docker/1.6.0/py3/Dockerfile.cpu

@@ -129,7 +129,7 @@ RUN pip install --no-cache-dir "sagemaker-pytorch-inference>=2"
 RUN curl https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.6.0/license.txt -o /license.txt
 
 RUN conda install -y -c conda-forge "pyyaml>5.4,<5.5"
-RUN pip install pillow==7.2.0 \
+RUN pip install pillow==8.2.0 \
     "awscli<2" \
     ruamel-yaml 
 

pytorch/inference/docker/1.6.0/py3/cu101/Dockerfile.gpu

@@ -140,7 +140,7 @@ RUN pip install --no-cache-dir "sagemaker-pytorch-inference>=2"
 RUN curl https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.6.0/license.txt -o /license.txt
 
 RUN conda install -y -c conda-forge "pyyaml>5.4,<5.5"
-RUN pip install pillow==7.2.0 "awscli<2"
+RUN pip install pillow==8.2.0 "awscli<2"
 
 RUN HOME_DIR=/root \
  && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \

pytorch/training/docker/1.6.0/py3/Dockerfile.cpu

@@ -113,7 +113,7 @@ RUN pip install --upgrade pip --trusted-host pypi.org --trusted-host files.pytho
     sagemaker-experiments==0.* \
     "sagemaker-pytorch-training<3" \
     psutil==5.6.7 \
-    Pillow==7.1.0 \
+    Pillow==8.2.0 \
  && pip install --no-cache-dir -U ${PT_TRAINING_URL} \
  && pip uninstall -y torchvision \
  && pip install --no-deps --no-cache-dir -U ${PT_TORCHVISION_URL}

pytorch/training/docker/1.8/py3/cu111/Dockerfile.gpu

@@ -5,9 +5,8 @@ LABEL dlc_major_version="1"
 
 ARG PYTHON=python3
 ARG PYTHON_VERSION=3.6.13
-ARG OPEN_MPI_VERSION=4.0.1
 ARG CUBLAS_VERSION=11.3.0.106
-ARG OPEN_MPI_PATH=/home/.openmpi
+ARG OPEN_MPI_PATH=/opt/amazon/openmpi/
 ARG CUDA_HOME=/usr/local/cuda
 ARG CONDA_PREFIX=/opt/conda
 ARG METIS=metis-5.1.0
@@ -30,8 +29,10 @@ ENV PATH /opt/conda/bin:$PATH
 ENV TORCH_CUDA_ARCH_LIST="3.7 5.0 7.0+PTX 8.0"
 ENV TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
 ENV CUDNN_VERSION=8.0.5.39
-ENV NCCL_VERSION=2.8.4
+ENV NCCL_VERSION=2.7.8
 ENV HOROVOD_VERSION=0.21.3
+ENV EFA_VERSION=1.11.2
+ENV BRANCH_OFI=1.1.1
 ENV DGLBACKEND=pytorch
 ENV CMAKE_PREFIX_PATH="$(dirname $(which conda))/../"
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_pytorch_container.training:main
@@ -40,7 +41,7 @@ ENV MANUAL_BUILD=0
 ARG PT_TRAINING_URL=https://aws-pytorch-binaries.s3-us-west-2.amazonaws.com/r1.8.1_aws/20210325-012734/e1343088f0beb99438343e1e99e8d71ffb972b47/gpu/torch-1.8.1-cp36-cp36m-manylinux1_x86_64.whl
 ARG PT_TORCHVISION_URL=https://torchvision-build.s3-us-west-2.amazonaws.com/1.8.1/gpu/torchvision-0.9.1-cp36-cp36m-linux_x86_64.whl
 ARG SMD_MODEL_PARALLEL_URL=https://sagemaker-distributed-model-parallel.s3.amazonaws.com/pytorch-1.8/build-artifacts/2021-03-26-22-01/smdistributed_modelparallel-1.3.1-cp36-cp36m-linux_x86_64.whl
-ARG SMDATAPARALLEL_BINARY=https://smdataparallel.s3.amazonaws.com/binary/pytorch/1.8.1/cu111/2021-04-01/smdistributed_dataparallel-1.1.1-cp36-cp36m-linux_x86_64.whl
+ARG SMDATAPARALLEL_BINARY=https://smdataparallel.s3.amazonaws.com/binary/pytorch/1.8.1/cu111/2021-04-16/smdistributed_dataparallel-1.2.0-cp36-cp36m-linux_x86_64.whl
 
 RUN apt-get update \
  && apt-get install -y --allow-change-held-packages --no-install-recommends \
@@ -63,8 +64,6 @@ RUN apt-get update \
     libcusparse-dev-11-1 \
     libglib2.0-0 \
     libgl1-mesa-glx \
-    libnccl2=${NCCL_VERSION}-1+cuda11.1 \
-    libnccl-dev=${NCCL_VERSION}-1+cuda11.1 \
     libsm6 \
     libxext6 \
     libxrender-dev \
@@ -75,22 +74,31 @@ RUN apt-get update \
     libnuma-dev \
     libssl1.1 \
     libtool \
+    hwloc \
     openssl \
     python3-dev \
     vim \
     wget \
     unzip \
     zlib1g-dev \
+    jq \
  && rm -rf /var/lib/apt/lists/*
 
-RUN wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-$OPEN_MPI_VERSION.tar.gz \
- && gunzip -c openmpi-$OPEN_MPI_VERSION.tar.gz | tar xf - \
- && cd openmpi-$OPEN_MPI_VERSION \
- && ./configure --prefix=$OPEN_MPI_PATH \
- && make all install \
- && cd .. \
- && rm openmpi-$OPEN_MPI_VERSION.tar.gz \
- && rm -rf openmpi-$OPEN_MPI_VERSION
+RUN cd /tmp \
+  && git clone https://github.com/NVIDIA/nccl.git -b v${NCCL_VERSION}-1 \
+  && cd nccl \
+  && make -j64 src.build BUILDDIR=/usr/local \
+  && rm -rf /tmp/nccl
+
+# Install EFA along with AWS OPEN_MPI
+RUN mkdir /tmp/efa \
+  && cd /tmp/efa \
+  && curl -O https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-${EFA_VERSION}.tar.gz \
+  && tar -xf aws-efa-installer-${EFA_VERSION}.tar.gz \
+  && cd aws-efa-installer \
+  && ./efa_installer.sh -y --skip-kmod -g \
+  && rm -rf /tmp/efa \
+  && rm -rf /tmp/aws-efa-installer-${EFA_VERSION}.tar.gz
 
 ENV PATH="$OPEN_MPI_PATH/bin:$PATH"
 ENV LD_LIBRARY_PATH="$OPEN_MPI_PATH/lib/:$LD_LIBRARY_PATH"
@@ -132,14 +140,15 @@ RUN conda install -c pytorch magma-cuda111==2.5.2 \
  && conda clean -ya
 
 # Install libboost from source. This package is needed for smdataparallel functionality [for networking asynchronous IO].
-RUN wget --quiet https://dl.bintray.com/boostorg/release/1.73.0/source/boost_1_73_0.tar.gz \
+RUN wget https://sourceforge.net/projects/boost/files/boost/1.73.0/boost_1_73_0.tar.gz/download -O boost_1_73_0.tar.gz \
   && tar -xzf boost_1_73_0.tar.gz \
   && cd boost_1_73_0 \
   && ./bootstrap.sh \
   && ./b2 threading=multi --prefix=${CONDA_PREFIX} -j 64 cxxflags=-fPIC cflags=-fPIC install || true \
   && cd .. \
   && rm -rf boost_1_73_0.tar.gz \
-  && rm -rf boost_1_73_0
+  && rm -rf boost_1_73_0 \
+  && cd ${CONDA_PREFIX}/include/boost
 
 WORKDIR /opt/pytorch
 
@@ -199,12 +208,6 @@ RUN  wget -nv https://github.com/rapidsai/rmm/archive/v${RMM_VERSION}.tar.gz \
   && rm -rf v${RMM_VERSION}.tar* \
   && rm -rf rmm-${RMM_VERSION}
 
-# Install Horovod
-RUN pip uninstall -y horovod \
- && ldconfig /usr/local/cuda-11.1/targets/x86_64-linux/lib/stubs \
- && HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_CUDA_HOME=/usr/local/cuda-11.1 HOROVOD_WITH_PYTORCH=1 pip install --no-cache-dir horovod==${HOROVOD_VERSION} \
- && ldconfig
-
 # Install Nvidia Apex
 RUN git clone https://github.com/NVIDIA/apex.git \
  && cd apex \
@@ -220,6 +223,21 @@ RUN mv $OPEN_MPI_PATH/bin/mpirun $OPEN_MPI_PATH/bin/mpirun.real \
  && echo NCCL_DEBUG=INFO >> /etc/nccl.conf \
  && echo NCCL_SOCKET_IFNAME=^docker0 >> /etc/nccl.conf
 
+# Install AWS OFI NCCL plug-in
+RUN apt-get update && apt-get install -y autoconf
+RUN mkdir /tmp/efa-ofi-nccl \
+  && cd /tmp/efa-ofi-nccl \
+  && git clone https://github.com/aws/aws-ofi-nccl.git -b v${BRANCH_OFI} \
+  && cd aws-ofi-nccl \
+  && ./autogen.sh \
+  && ./configure --with-libfabric=/opt/amazon/efa \
+  --with-mpi=/opt/amazon/openmpi \
+  --with-cuda=/usr/local/cuda \
+  --with-nccl=/usr/local --prefix=/usr/local \
+  && make \
+  && make install \
+  && rm -rf /tmp/efa-ofi-nccl
+
 # Install OpenSSH for MPI to communicate between containers, allow OpenSSH to talk to containers without asking for confirmation
 RUN apt-get update \
  && apt-get install -y  --allow-downgrades --allow-change-held-packages --no-install-recommends \
@@ -240,6 +258,12 @@ RUN rm -rf /root/.ssh/ && \
  cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \
  && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config
 
+# Install Horovod
+RUN pip uninstall -y horovod \
+ && ldconfig /usr/local/cuda-11.1/targets/x86_64-linux/lib/stubs \
+ && HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_CUDA_HOME=/usr/local/cuda-11.1 HOROVOD_WITH_PYTORCH=1 pip install --no-cache-dir horovod==${HOROVOD_VERSION} \
+ && ldconfig
+
 # Install SM Distributed Modelparallel binary
 RUN pip install --no-cache-dir -U ${SMD_MODEL_PARALLEL_URL}
 

release_images.yml

@@ -330,25 +330,22 @@ release_images:
       example: False
       disable_sm_tag: False  # [Default: False] This option is not used by Example images
       force_release: False
-  23: # PT-1.6.0 Wave-2
+  23:
     framework: "pytorch"
     version: "1.6.0"
     training:
-      device_types: ["gpu"]
+      device_types: ["cpu"]
       python_versions: ["py36"]
-      os_version: "ubuntu18.04"
-      cuda_version: "cu110"
+      os_version: "ubuntu16.04"
+      cuda_version: "cu101"
       example: False
-      disable_sm_tag: True  # [Default: False] This option is not used by Example images
+      disable_sm_tag: False  # [Default: False] This option is not used by Example images
       force_release: False
-  24: # PT-1.6.0 Wave-2
-    framework: "pytorch"
-    version: "1.6.0"
-    training:
-      device_types: ["gpu"]
+    inference:
+      device_types: ["cpu", "gpu"]
       python_versions: ["py36"]
-      os_version: "ubuntu18.04"
-      cuda_version: "cu110"
-      example: True
+      os_version: "ubuntu16.04"
+      cuda_version: "cu101"
+      example: False
       disable_sm_tag: False  # [Default: False] This option is not used by Example images
       force_release: False

src/config/test_config.py

@@ -1,7 +1,12 @@
 # Please only set it to True if you are preparing a Benchmark related PR
 # Do remember to revert it back to False before merging any PR (including Benchmark dedicated PR)
 ENABLE_BENCHMARK_DEV_MODE = False
+
 # Disable the test codebuild jobs to be run
+
+# It is recommended to set DISABLE_EFA_TESTS to True to disable EFA tests if there is no change to EFA installer version or Frameworks.
+DISABLE_EFA_TESTS = False
+
 DISABLE_SANITY_TESTS = False
 DISABLE_SAGEMAKER_TESTS = False
 DISABLE_ECS_TESTS = False

src/deep_learning_container.py

@@ -209,12 +209,10 @@ def tag_instance():
     request_status = None
     if instance_id and region:
         try:
-            # # The section below has been commented out because the feature has been disabled until it is
-            # # ready to be enabled.
-            # session = botocore.session.get_session()
-            # ec2_client = session.create_client("ec2", region_name=region)
-            # response = ec2_client.create_tags(Resources=[instance_id], Tags=[tag_struct])
-            # request_status = response.get("ResponseMetadata").get("HTTPStatusCode")
+            session = botocore.session.get_session()
+            ec2_client = session.create_client("ec2", region_name=region)
+            response = ec2_client.create_tags(Resources=[instance_id], Tags=[tag_struct])
+            request_status = response.get("ResponseMetadata").get("HTTPStatusCode")
             if os.environ.get("TEST_MODE") == str(1):
                 with open(os.path.join(os.sep, "tmp", "test_tag_request.txt"), "w+") as rf:
                     rf.write(json.dumps(tag_struct, indent=4))

src/start_testbuilds.py

@@ -47,7 +47,8 @@ def run_test_job(commit, codebuild_project, images_str=""):
             {"name": "PR_NUMBER", "value": pr_num, "type": "PLAINTEXT"},
             # USE_SCHEDULER is passed as an env variable here because it is more convenient to set this in
             # config/test_config, compared to having another config file under dlc/tests/.
-            {"name": "USE_SCHEDULER", "value": str(test_config.USE_SCHEDULER), "type": "PLAINTEXT"}
+            {"name": "USE_SCHEDULER", "value": str(test_config.USE_SCHEDULER), "type": "PLAINTEXT"},
+            {"name": "DISABLE_EFA_TESTS", "value": str(test_config.DISABLE_EFA_TESTS), "type": "PLAINTEXT"},
         ]
     )
     LOGGER.debug(f"env_overrides dict: {env_overrides}")

tensorflow/training/docker/2.4/py3/cu110/Dockerfile.gpu

@@ -26,7 +26,10 @@ ARG PYTHON=python3.7
 ARG PYTHON_PIP=python3-pip
 ARG PIP=pip3
 ARG PYTHON_VERSION=3.7.10
-ARG OPEN_MPI_PATH=/usr/local
+ARG OPEN_MPI_PATH=/opt/amazon/openmpi/
+ARG NCCL_VERSION=2.7.8
+ARG EFA_VERSION=1.11.2
+ARG BRANCH_OFI=1.1.1
 
 ARG TF_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/tensorflow/r2.4_aws/20210127-150238/gpu/py37/cu110/tensorflow_gpu-2.4.1-cp37-cp37m-manylinux2010_x86_64.whl
 
@@ -36,7 +39,7 @@ ARG ESTIMATOR_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/est
 # the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed.
 ARG SMDEBUG_VERSION=1.0.8
 
-ARG SMDATAPARALLEL_BINARY=https://smdataparallel.s3.amazonaws.com/binary/tensorflow/2.4.1/cu110/2021-04-08/smdistributed_dataparallel-1.1.2-cp37-cp37m-linux_x86_64.whl
+ARG SMDATAPARALLEL_BINARY=https://smdataparallel.s3.amazonaws.com/binary/tensorflow/2.4.1/cu110/2021-04-12/smdistributed_dataparallel-1.2.0-cp37-cp37m-linux_x86_64.whl
 
 ARG SMMODELPARALLEL_BINARY=https://sagemaker-distributed-model-parallel.s3.amazonaws.com/tensorflow-2.4/build-artifacts/2021-03-26-21-57/smdistributed_modelparallel-1.3.1-cp37-cp37m-linux_x86_64.whl
 
@@ -52,15 +55,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
     emacs \
     libcudnn8=8.0.5.39-1+cuda11.0 \
     # TensorFlow doesn't require libnccl anymore but Open MPI still depends on it
-    libnccl2=2.7.8-1+cuda11.0 \
     libgomp1 \
-    libnccl-dev=2.7.8-1+cuda11.0 \
     libfreetype6-dev \
     libhdf5-serial-dev \
     liblzma-dev \
     libpng-dev \
     libtemplate-perl \
     libzmq3-dev \
+    hwloc \
     git \
     unzip \
     wget \
@@ -74,6 +76,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
     zlib1g-dev \
     # Install dependent library for OpenCV
     libgtk2.0-dev \
+    jq \
  && apt-get update \
  && apt-get install -y --no-install-recommends --allow-unauthenticated \
     libcublas-11-0=11.2.0.252-1 \
@@ -93,31 +96,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
  && rm -rf /var/lib/apt/lists/* \
  && mkdir -p /var/run/sshd
 
-RUN wget --quiet https://dl.bintray.com/boostorg/release/1.73.0/source/boost_1_73_0.tar.gz \
+RUN cd /tmp \
+  && git clone https://github.com/NVIDIA/nccl.git -b v$NCCL_VERSION-1 \
+  && cd nccl \
+  && make -j64 src.build BUILDDIR=/usr/local \
+  && rm -rf /tmp/nccl
+
+# Install EFA along with AWS OPEN_MPI
+RUN mkdir /tmp/efa \
+  && cd /tmp/efa \
+  && curl -O https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-$EFA_VERSION.tar.gz \
+  && tar -xf aws-efa-installer-$EFA_VERSION.tar.gz \
+  && cd aws-efa-installer \
+  && ./efa_installer.sh -y --skip-kmod -g \
+  && rm -rf /tmp/efa \
+  && rm -rf /tmp/aws-efa-installer-${EFA_VERSION}.tar.gz
+
+RUN wget https://sourceforge.net/projects/boost/files/boost/1.73.0/boost_1_73_0.tar.gz/download -O boost_1_73_0.tar.gz \
  && tar -xzf boost_1_73_0.tar.gz \
  && cd boost_1_73_0 \
  && ./bootstrap.sh \
  && ./b2 threading=multi --prefix=/usr -j 64 cxxflags=-fPIC cflags=-fPIC install || true \
  && cd .. \
  && rm -rf boost_1_73_0.tar.gz \
- && rm -rf boost_1_73_0
+ && rm -rf boost_1_73_0 \
+ && cd /usr/include/boost
 
 ###########################################################################
 # Horovod & its dependencies
 ###########################################################################
 
-# Install Open MPI
-RUN mkdir /tmp/openmpi \
- && cd /tmp/openmpi \
- && curl -fSsL -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.4.tar.gz \
- && tar zxf openmpi-4.0.4.tar.gz \
- && cd openmpi-4.0.4 \
- && ./configure --enable-orterun-prefix-by-default \
- && make -j $(nproc) all \
- && make install \
- && ldconfig \
- && rm -rf /tmp/openmpi
-
 # Create a wrapper for OpenMPI to allow running as root by default
 RUN mv $OPEN_MPI_PATH/bin/mpirun $OPEN_MPI_PATH/bin/mpirun.real \
  && echo '#!/bin/bash' > $OPEN_MPI_PATH/bin/mpirun \
@@ -132,10 +140,10 @@ RUN echo "hwloc_base_binding_policy = none" >> $OPEN_MPI_PATH/etc/openmpi-mca-pa
 # Set default NCCL parameters
 RUN echo NCCL_DEBUG=INFO >> /etc/nccl.conf
 
-ENV LD_LIBRARY_PATH=$OPEN_MPI_PATH/openmpi/lib:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH=$OPEN_MPI_PATH/lib:$LD_LIBRARY_PATH
 # /usr/local/lib/libpython* needs to be accessible for dynamic linking
 ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
-ENV PATH=$OPEN_MPI_PATH/openmpi/bin/:$PATH
+ENV PATH=$OPEN_MPI_PATH/bin/:$PATH
 ENV PATH=$OPEN_MPI_PATH/nvidia/bin:$PATH
 
 # SSH login fix. Otherwise user is kicked off after login
@@ -238,6 +246,21 @@ RUN  wget -nv https://github.com/rapidsai/rmm/archive/v${RMM_VERSION}.tar.gz \
   && rm -rf v${RMM_VERSION}.tar* \
   && rm -rf rmm-${RMM_VERSION}
 
+# Install AWS OFI NCCL plug-in
+RUN apt-get update && apt-get install -y autoconf
+RUN mkdir /tmp/efa-ofi-nccl \
+  && cd /tmp/efa-ofi-nccl \
+  && git clone https://github.com/aws/aws-ofi-nccl.git -b v$BRANCH_OFI \
+  && cd aws-ofi-nccl \
+  && ./autogen.sh \
+  && ./configure --with-libfabric=/opt/amazon/efa \
+  --with-mpi=/opt/amazon/openmpi \
+  --with-cuda=/usr/local/cuda \
+  --with-nccl=/usr/local --prefix=/usr/local \
+  && make \
+  && make install \
+  && rm -rf /tmp/efa-ofi-nccl
+
 ENV CPATH="/usr/local/lib/python3.7/dist-packages/pybind11/include/"
 
 RUN apt-get update && apt-get -y install cmake protobuf-compiler

test/__init__.py

@@ -1 +0,0 @@
-