diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..04e65d0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,64 @@ +*.7zip +*.a +*.bak +build/ +_build/ +build/ +*.bz +*.bz2 +*.conda +*.core +*.coverage +*.css +*.csv +*.dat +*.db +dist/ +*.dll +*.doc +*.docx +*.docm +downloads/ +.DS_Store +*.dylib +*.egg-info/ +*.env +*.exe +*.feather +*.html +htmlcov/ +.idea/ +*.js +*.json +.lycheecache +*.lzma +.mypy_cache/ +*.npy +*.o +*.pdf +*.pem +*.ppt +*.pptx +*.pptm +*.pq +*.pub +*.pyc +__pycache__/ +.pytest_cache/ +*.rda +*.rds +*.Rdata +*.rsa +.ruff_cache/ +*.snappy-*.tar.gz +*.so +*.sqlite +*.tar.gz +*.tgz +*.tmp +*.whl +*.xls +*.xlsx +*.xlsm +*.zip +*.zstd diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93fce75..3228a33 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,17 +8,21 @@ We love your input! We want to make contributing to this project as easy and tra - Proposing new features ## We Develop with GitHub + We use GitHub to host code, track issues and feature requests, and accept pull requests. ## Report a bug or request a feature + The best way to get in touch is by [opening an issue](https://github.com/rapidsai/ci-imgs/issues/new/choose). ## To make a code contribution + 1. Fork the repo and create your branch from `main`. 2. Make your changes. 3. Create a pull request! ## Developer Certificate of Origin + All contributions to this project must be accompanied by a sign-off indicating that the contribution is made pursuant to the Developer Certificate of Origin (DCO). This is a lightweight way for contributors to certify that they wrote or otherwise have the right to submit the code they are contributing to the project. The DCO is a simple statement that you, as a contributor, have the legal right to make the contribution. To certify your adherence to the DCO, you must sign off on your commits. This is done by adding a `Signed-off-by` line to your commit messages: @@ -60,4 +64,5 @@ By making a contribution to this project, I certify that: ``` ## Attribution + Portions adopted from https://github.com/pytorch/pytorch/blob/main/CONTRIBUTING.md diff --git a/README.md b/README.md index 40f9675..8df03a7 100644 --- a/README.md +++ b/README.md @@ -55,9 +55,17 @@ pull in bug fixes, new features, etc. without needing to manually update tags as The `latest` image tags are controlled by the values in `latest.yaml`. -## Building the dockerfiles locally +## Building the images locally -To build the dockerfiles locally, you may use the following snippets. +To build the images locally, you may use the following snippets. + +These scripts require the `gha-tools` project. +If you don't have it installed, you may install it like this: + +```shell +git clone https://github.com/rapidsai/gha-tools.git /tmp/gha-tools +export PATH="/tmp/gha-tools/tools:${PATH}" +``` The `ci-conda` and `ci-wheel` images require a GitHub token to download sccache releases. If you have the `gh` CLI installed and authenticated, you can use `gh auth token` to get your token: diff --git a/ci-conda.Dockerfile b/ci-conda.Dockerfile index ddae4f7..166f9ca 100644 --- a/ci-conda.Dockerfile +++ b/ci-conda.Dockerfile @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 ################################ build and update miniforge-upstream ############################### @@ -19,13 +19,13 @@ SHELL ["/bin/bash", "-euo", "pipefail", "-c"] ARG SCCACHE_VER=notset ARG GH_CLI_VER=notset ARG CPU_ARCH=notset -RUN --mount=type=secret,id=GH_TOKEN <= 5)) && break; sleep 10; done apt-get install -y --no-install-recommends wget wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin wget -q https://github.com/cli/cli/releases/download/v${GH_CLI_VER}/gh_${GH_CLI_VER}_linux_${CPU_ARCH}.tar.gz tar -xf gh_*.tar.gz && mv gh_*/bin/gh /usr/local/bin && rm -rf gh_* - GH_TOKEN=$(cat /run/secrets/GH_TOKEN) SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache + SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache apt-get purge -y wget && apt-get autoremove -y rm -rf /var/lib/apt/lists/* EOF @@ -74,7 +74,7 @@ EOF ARG SCCACHE_VER=notset ARG GH_CLI_VER=notset ARG CPU_ARCH=notset -RUN --mount=type=secret,id=GH_TOKEN <= 5)) && break; sleep 10; done @@ -82,7 +82,7 @@ case "${LINUX_VER}" in wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin wget -q https://github.com/cli/cli/releases/download/v${GH_CLI_VER}/gh_${GH_CLI_VER}_linux_${CPU_ARCH}.tar.gz tar -xf gh_*.tar.gz && mv gh_*/bin/gh /usr/local/bin && rm -rf gh_* - GH_TOKEN=$(cat /run/secrets/GH_TOKEN) SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache + SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache apt-get purge -y wget && apt-get autoremove -y rm -rf /var/lib/apt/lists/* ;; @@ -91,7 +91,7 @@ case "${LINUX_VER}" in wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin wget -q https://github.com/cli/cli/releases/download/v${GH_CLI_VER}/gh_${GH_CLI_VER}_linux_${CPU_ARCH}.tar.gz tar -xf gh_*.tar.gz && mv gh_*/bin/gh /usr/local/bin && rm -rf gh_* - GH_TOKEN=$(cat /run/secrets/GH_TOKEN) SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache + SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache dnf remove -y wget dnf clean all ;; @@ -151,22 +151,24 @@ echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> /etc/skel/.ba echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> ~/.bashrc EOF -# tzdata is needed by the ORC library used by pyarrow, because it provides /etc/localtime -# On Ubuntu 24.04 and newer, we also need tzdata-legacy RUN < "24.04" ]] || [[ "${os_version}" == "24.04" ]]; then - tzdata_pkgs=(tzdata tzdata-legacy) - else - tzdata_pkgs=(tzdata) + PACKAGES_TO_INSTALL+=(tzdata-legacy) fi rapids-retry apt-get update -y apt-get upgrade -y apt-get install -y --no-install-recommends \ - "${tzdata_pkgs[@]}" + "${PACKAGES_TO_INSTALL[@]}" rm -rf "/var/lib/apt/lists/*" ;; @@ -205,28 +207,34 @@ case "${LINUX_VER}" in "ubuntu"*) rapids-retry apt-get update -y apt-get upgrade -y - apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - file \ - unzip \ - wget \ - gcc \ + PACKAGES_TO_INSTALL=( + ca-certificates + curl + file + unzip + wget + gcc g++ + ) + apt-get install -y --no-install-recommends \ + "${PACKAGES_TO_INSTALL[@]}" update-ca-certificates rm -rf /var/cache/apt/archives /var/lib/apt/lists/* ;; "rockylinux"*) dnf -y update - dnf -y install --setopt=install_weak_deps=False \ - ca-certificates \ - file \ - unzip \ - wget \ - which \ - yum-utils \ - gcc \ + PACKAGES_TO_INSTALL=( + ca-certificates + file + unzip + wget + which + yum-utils + gcc gcc-c++ + ) + dnf -y install --setopt=install_weak_deps=False \ + "${PACKAGES_TO_INSTALL[@]}" update-ca-trust extract dnf clean all ;; @@ -264,20 +272,24 @@ else PYTHON_ABI_TAG="cpython" fi +PACKAGES_TO_INSTALL=( + 'anaconda-client>=1.13.1' + 'ca-certificates>=2026.1.4' + 'certifi>=2026.1.4' + 'conda-build>=25.11.1' + 'conda-package-handling>=2.4.0' + 'dunamai>=1.25.0' + 'git>=2.52.0' + 'jq>=1.8.1' + 'packaging>=25.0' + "python>=${PYTHON_VERSION},<${PYTHON_UPPER_BOUND}=*_${PYTHON_ABI_TAG}" + 'rapids-dependency-file-generator==1.*' + 'rattler-build>=0.55.0' +) + rapids-mamba-retry install -y \ - anaconda-client \ - ca-certificates \ - certifi \ - conda-build \ - conda-package-handling \ - dunamai \ - git \ - jq \ - packaging \ - "python>=${PYTHON_VERSION},<${PYTHON_UPPER_BOUND}=*_${PYTHON_ABI_TAG}" \ - "rapids-dependency-file-generator==1.*" \ - rattler-build \ -; + "${PACKAGES_TO_INSTALL[@]}" + conda clean -aiptfy EOF diff --git a/ci-wheel.Dockerfile b/ci-wheel.Dockerfile index bb2734d..8b7b9ac 100644 --- a/ci-wheel.Dockerfile +++ b/ci-wheel.Dockerfile @@ -49,7 +49,7 @@ EOF # NOTE: gh CLI must be installed before rapids-install-sccache (uses `gh release download`) ARG SCCACHE_VER=notset ARG GH_CLI_VER=notset -RUN --mount=type=secret,id=GH_TOKEN <= 5)) && break; sleep 10; done @@ -57,7 +57,7 @@ case "${LINUX_VER}" in wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin wget -q https://github.com/cli/cli/releases/download/v${GH_CLI_VER}/gh_${GH_CLI_VER}_linux_${CPU_ARCH}.tar.gz tar -xf gh_*.tar.gz && mv gh_*/bin/gh /usr/local/bin && rm -rf gh_* - GH_TOKEN=$(cat /run/secrets/GH_TOKEN) SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache + SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache apt-get purge -y wget && apt-get autoremove -y rm -rf /var/lib/apt/lists/* ;; @@ -66,7 +66,7 @@ case "${LINUX_VER}" in wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin wget -q https://github.com/cli/cli/releases/download/v${GH_CLI_VER}/gh_${GH_CLI_VER}_linux_${CPU_ARCH}.tar.gz tar -xf gh_*.tar.gz && mv gh_*/bin/gh /usr/local/bin && rm -rf gh_* - GH_TOKEN=$(cat /run/secrets/GH_TOKEN) SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache + SCCACHE_VERSION="${SCCACHE_VER}" rapids-install-sccache dnf remove -y wget dnf clean all ;; @@ -81,7 +81,7 @@ RUN <&1 >/dev/null; then echo "libnccl-dev not found, manually installing it" - LIBRARIES_TO_INSTALL+=(libnccl-dev) + PACKAGES_TO_INSTALL+=(libnccl-dev) else echo "libnccl-dev already installed" fi apt-get install -y --no-install-recommends \ - "${LIBRARIES_TO_INSTALL[@]}" + "${PACKAGES_TO_INSTALL[@]}" update-ca-certificates add-apt-repository ppa:git-core/ppa @@ -141,7 +141,7 @@ case "${LINUX_VER}" in dnf update -y dnf install -y epel-release dnf update -y - LIBRARIES_TO_INSTALL=( + PACKAGES_TO_INSTALL=( autoconf automake bzip2 @@ -179,13 +179,13 @@ case "${LINUX_VER}" in # only re-install NCCL if there wasn't one already installed in the image if ! rpm --query --all | grep -E 'libnccl\-devel' > /dev/null 2>&1; then echo "libnccl-devel not found, manually installing it" - LIBRARIES_TO_INSTALL+=(libnccl-devel) + PACKAGES_TO_INSTALL+=(libnccl-devel) else echo "libnccl-devel already installed" fi dnf install -y \ - "${LIBRARIES_TO_INSTALL[@]}" + "${PACKAGES_TO_INSTALL[@]}" update-ca-trust extract dnf config-manager --set-enabled powertools dnf install -y blas-devel lapack-devel @@ -251,17 +251,21 @@ pyenv global ${PYTHON_VER} # `rapids-pip-retry` defaults to using `python -m pip` to select which `pip` to # use so should be compatible with `pyenv` rapids-pip-retry install --upgrade pip + +PACKAGES_TO_INSTALL=( + 'anaconda-client>=1.13.0' + 'auditwheel>=6.2.0' + 'certifi>=2026.1.4' + 'conda-package-handling>=2.4.0' + 'dunamai>=1.25.0' + 'patchelf>=0.17.2.4' + 'pydistcheck==0.11.*' + 'rapids-dependency-file-generator==1.*' + 'twine>=6.2.0' + 'wheel>=0.45.1' +) rapids-pip-retry install \ - 'anaconda-client>=1.13.0' \ - 'auditwheel>=6.2.0' \ - certifi \ - conda-package-handling \ - dunamai \ - patchelf \ - 'pydistcheck==0.9.*' \ - 'rapids-dependency-file-generator==1.*' \ - twine \ - wheel + "${PACKAGES_TO_INSTALL[@]}" pip cache purge pyenv rehash EOF diff --git a/citestwheel.Dockerfile b/citestwheel.Dockerfile index 6a0d3cf..881682d 100644 --- a/citestwheel.Dockerfile +++ b/citestwheel.Dockerfile @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 ARG CUDA_VER=notset @@ -75,41 +75,44 @@ case "${LINUX_VER}" in rapids-retry apt-get update -y apt-get upgrade -y + PACKAGES_TO_INSTALL=( + build-essential + ca-certificates + curl + git + jq + libbz2-dev + libffi-dev + liblzma-dev + libncursesw5-dev + libnuma1 + libreadline-dev + libsqlite3-dev + libssl-dev + libxml2-dev + libxmlsec1-dev + llvm + make + patch + ssh + tk-dev + tzdata + unzip + wget + xz-utils + zlib1g-dev + ) + # tzdata is needed by the ORC library used by pyarrow, because it provides /etc/localtime # On Ubuntu 24.04 and newer, we also need tzdata-legacy os_version=$(grep 'VERSION_ID' /etc/os-release | cut -d '"' -f 2) if [[ "${os_version}" > "24.04" ]] || [[ "${os_version}" == "24.04" ]]; then - tzdata_pkgs=(tzdata tzdata-legacy) - else - tzdata_pkgs=(tzdata) + PACKAGES_TO_INSTALL+=(tzdata-legacy) fi apt-get install -y --no-install-recommends \ - "${tzdata_pkgs[@]}" \ - build-essential \ - ca-certificates \ - curl \ - git \ - jq \ - libbz2-dev \ - libffi-dev \ - liblzma-dev \ - libncursesw5-dev \ - libnuma1 \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - libxml2-dev \ - libxmlsec1-dev \ - llvm \ - make \ - patch \ - ssh \ - tk-dev \ - unzip \ - wget \ - xz-utils \ - zlib1g-dev + "${PACKAGES_TO_INSTALL[@]}" \ + update-ca-certificates rm -rf /var/cache/apt/archives /var/lib/apt/lists/* @@ -118,27 +121,31 @@ case "${LINUX_VER}" in dnf update -y dnf install -y epel-release dnf update -y - dnf install -y \ - bzip2 \ - bzip2-devel \ - ca-certificates \ - curl \ - dnf-plugins-core \ - gcc \ - git \ - jq \ - libffi-devel \ - patch \ - ncurses-devel \ - readline-devel \ - sqlite \ - sqlite-devel \ - unzip \ - wget \ - which \ - xz \ - xz-devel \ + PACKAGES_TO_INSTALL=( + bzip2 + bzip2-devel + ca-certificates + curl + dnf-plugins-core + gcc + git + jq + libffi-devel + patch + ncurses-devel + readline-devel + sqlite + sqlite-devel + unzip + wget + which + xz + xz-devel zlib-devel + ) + dnf install -y \ + "${PACKAGES_TO_INSTALL[@]}" + update-ca-trust extract dnf clean all pushd tmp @@ -199,7 +206,7 @@ pyenv global ${PYTHON_VER} # use so should be compatible with `pyenv` rapids-pip-retry install --upgrade pip rapids-pip-retry install \ - certifi \ + 'certifi>=2026.1.4' \ 'rapids-dependency-file-generator==1.*' pyenv rehash EOF