From a0a48542bc25420ce0719dc06febce4e7b9f49e2 Mon Sep 17 00:00:00 2001 From: Nesh Devanathan Date: Thu, 17 Aug 2023 00:58:21 -0400 Subject: [PATCH 1/8] Reorder cupy to ensure that cupy is installed with respective cuda version. The base docker image contains two cuda installations which is causing errors for `cupy` package. --- Dockerfile.tmpl | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 5597ff59..7eeb3fdc 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -99,6 +99,14 @@ RUN conda config --add channels nvidia && \ mamba install -y mkl cartopy imagemagick pyproj "shapely<2" && \ /tmp/clean-layer.sh +# Install spacy +{{ if eq .Accelerator "gpu" }} +RUN mamba install -y -c conda-forge spacy cupy cuda-version=11.8 && \ + /tmp/clean-layer.sh +{{ else }} +RUN pip install spacy && \ + /tmp/clean-layer.sh +{{ end}} {{ if eq .Accelerator "gpu" }} # b/232247930: uninstall pyarrow to avoid double installation with the GPU specific version. @@ -126,6 +134,7 @@ RUN pip install \ /tmp/clean-layer.sh {{ end }} + # Install LightGBM {{ if eq .Accelerator "gpu" }} COPY --from=lightgbm_whl /tmp/whl/*.whl /tmp/lightgbm/ @@ -150,14 +159,6 @@ RUN pip install jax[cpu] && \ /tmp/clean-layer.sh {{ end }} -# Install spacy -{{ if eq .Accelerator "gpu" }} -RUN mamba install -y -c conda-forge spacy cupy && \ - /tmp/clean-layer.sh -{{ else }} -RUN pip install spacy && \ - /tmp/clean-layer.sh -{{ end}} # Install GPU specific packages {{ if eq .Accelerator "gpu" }} @@ -678,4 +679,4 @@ RUN echo "$GIT_COMMIT" > /etc/git_commit && echo "$BUILD_DATE" > /etc/build_date {{ if eq .Accelerator "gpu" }} # Remove the CUDA stubs. ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH_NO_STUBS" -{{ end }} +{{ end }} \ No newline at end of file From 5833c37a015b627ee314fca2a2b4117e59d2cd9f Mon Sep 17 00:00:00 2001 From: Nesh Devanathan Date: Thu, 17 Aug 2023 13:44:59 -0400 Subject: [PATCH 2/8] Use variables for major and minor version of cuda --- Dockerfile.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 7eeb3fdc..226cd237 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -101,7 +101,7 @@ RUN conda config --add channels nvidia && \ # Install spacy {{ if eq .Accelerator "gpu" }} -RUN mamba install -y -c conda-forge spacy cupy cuda-version=11.8 && \ +RUN mamba install -y -c conda-forge spacy cupy cuda-version=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION && \ /tmp/clean-layer.sh {{ else }} RUN pip install spacy && \ @@ -134,7 +134,6 @@ RUN pip install \ /tmp/clean-layer.sh {{ end }} - # Install LightGBM {{ if eq .Accelerator "gpu" }} COPY --from=lightgbm_whl /tmp/whl/*.whl /tmp/lightgbm/ From d9d1e25c8d59d94bbfc74c015e3eacb77739b51e Mon Sep 17 00:00:00 2001 From: Nesh Devanathan Date: Thu, 17 Aug 2023 15:25:50 -0400 Subject: [PATCH 3/8] Added fix for failing test cases `cudf/api/types.py` has nominal impact to other packages. --- Dockerfile.tmpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 226cd237..c4df4612 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -113,6 +113,12 @@ RUN pip install spacy && \ RUN pip uninstall -y pyarrow && \ mamba install -y cudf cuml && \ /tmp/clean-layer.sh + +# TODO(neshdev): Resolve pandas dependency another way +RUN cat /opt/conda/lib/python3.10/site-packages/cudf/api/types.py \ + | sed 's/^is_extension_type/# is_extension_type/g' \ + | sed 's/^is_categorical/# is_categorical/g' \ + > /opt/conda/lib/python3.10/site-packages/cudf/api/types.py {{ end }} # Install PyTorch From 213a24aaacb0c6150bc9934f18088e26ec3714db Mon Sep 17 00:00:00 2001 From: Nesh Devanathan Date: Thu, 17 Aug 2023 15:44:58 -0400 Subject: [PATCH 4/8] Change sed command to update the file inplace. pipe-ing to the file was causing an empty file. --- Dockerfile.tmpl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index c4df4612..38c87e69 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -114,11 +114,9 @@ RUN pip uninstall -y pyarrow && \ mamba install -y cudf cuml && \ /tmp/clean-layer.sh -# TODO(neshdev): Resolve pandas dependency another way -RUN cat /opt/conda/lib/python3.10/site-packages/cudf/api/types.py \ - | sed 's/^is_extension_type/# is_extension_type/g' \ - | sed 's/^is_categorical/# is_categorical/g' \ - > /opt/conda/lib/python3.10/site-packages/cudf/api/types.py +# TODO: b/296444923 - Resolve pandas dependency another way +RUN sed -i 's/^is_extension_type/# is_extension_type/g' /opt/conda/lib/python3.10/site-packages/cudf/api/types.py \ + && sed -i 's/^is_categorical/# is_categorical/g' /opt/conda/lib/python3.10/site-packages/cudf/api/types.py {{ end }} # Install PyTorch From 226ac688021162bc18d1f20db5ea0b192ad195b1 Mon Sep 17 00:00:00 2001 From: Nesh Devanathan Date: Thu, 17 Aug 2023 18:33:24 -0400 Subject: [PATCH 5/8] Change H2O to resolve via conda and bundle tf into its own layer --- Dockerfile.tmpl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 38c87e69..e4f54547 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -183,10 +183,8 @@ RUN JAXVER=$(pip freeze | grep -e "^jax==") && \ flax \ "${JAXVER}" && \ - # Install h2o from source. - # Use `conda install -c h2oai h2o` once Python 3.7 version is released to conda. - apt-get install -y default-jre-headless && \ - pip install -f https://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o \ +RUN conda install -c h2oai h2o && /tmp/clean-layer.sh +RUN pip install \ "tensorflow-gcs-config<=${TENSORFLOW_VERSION}" \ "tensorflow==${TENSORFLOW_VERSION}" \ tensorflow-addons \ From 5ebeb5551523c0bdd0e0b43bca54dde0cf7a492e Mon Sep 17 00:00:00 2001 From: Nesh Devanathan Date: Fri, 18 Aug 2023 01:15:50 +0000 Subject: [PATCH 6/8] Fix syntax error when trying to partition pacakge --- Dockerfile.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index e4f54547..301464a1 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -181,9 +181,10 @@ RUN JAXVER=$(pip freeze | grep -e "^jax==") && \ pandas \ polars \ flax \ - "${JAXVER}" && \ + "${JAXVER}" RUN conda install -c h2oai h2o && /tmp/clean-layer.sh + RUN pip install \ "tensorflow-gcs-config<=${TENSORFLOW_VERSION}" \ "tensorflow==${TENSORFLOW_VERSION}" \ From 911b5082e40402285a7f798359762eb8dc5ba43d Mon Sep 17 00:00:00 2001 From: Nesh Devanathan Date: Fri, 18 Aug 2023 02:35:15 +0000 Subject: [PATCH 7/8] h20 for python3.10 is unavailable in conda. Use pip installation. --- Dockerfile.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 301464a1..a5cb8ef1 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -183,7 +183,7 @@ RUN JAXVER=$(pip freeze | grep -e "^jax==") && \ flax \ "${JAXVER}" -RUN conda install -c h2oai h2o && /tmp/clean-layer.sh +RUN pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o && /tmp/clean-layer.sh RUN pip install \ "tensorflow-gcs-config<=${TENSORFLOW_VERSION}" \ From 1318e3374920defb10450a3ef27dd10a4a372b40 Mon Sep 17 00:00:00 2001 From: Nesh Devanathan Date: Fri, 18 Aug 2023 04:01:20 +0000 Subject: [PATCH 8/8] h2o package needs java jre --- Dockerfile.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index a5cb8ef1..4d75a47a 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -183,6 +183,8 @@ RUN JAXVER=$(pip freeze | grep -e "^jax==") && \ flax \ "${JAXVER}" +RUN apt-get install -y default-jre + RUN pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o && /tmp/clean-layer.sh RUN pip install \ @@ -251,7 +253,6 @@ RUN pip install scipy \ datashader \ # Boruta (python implementation) Boruta && \ - apt-get install -y graphviz && pip install graphviz && \ # Pandoc is a dependency of deap apt-get install -y pandoc && \ @@ -473,6 +474,7 @@ RUN pip install bleach \ pyarrow \ feather-format \ fastai + RUN python -m spacy download en_core_web_sm && python -m spacy download en_core_web_lg && \ apt-get update && apt-get install -y ffmpeg && \ /tmp/clean-layer.sh