rapidsai · trxcllnt · Mar 7, 2024 · Jan 9, 2024 · Jan 9, 2024 · Jan 10, 2024
diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json
@@ -10,9 +10,11 @@
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
+    "./features/src/utils": {},
     "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
+    "./features/src/utils",
     "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
@@ -42,9 +44,30 @@
   "customizations": {
     "vscode": {
       "extensions": [
+        "augustocdias.tasks-shell-input",
         "ms-python.flake8",
         "nvidia.nsight-vscode-edition"
-      ]
+      ],
+      "files.watcherExclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/target/**": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      },
+      "search.exclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/*.code-search": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      }
     }
   }
 }
diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json
@@ -12,11 +12,13 @@
   "features": {
     "./features/src/ucx": {"version": "1.14.1"},
     "./features/src/cuda": {"version": "11.8", "installcuBLAS": true, "installcuDNN": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true},
+    "./features/src/utils": {},
     "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
     "./features/src/ucx",
     "./features/src/cuda",
+    "./features/src/utils",
     "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
@@ -45,9 +47,30 @@
   "customizations": {
     "vscode": {
       "extensions": [
+        "augustocdias.tasks-shell-input",
         "ms-python.flake8",
         "nvidia.nsight-vscode-edition"
-      ]
+      ],
+      "files.watcherExclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/target/**": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      },
+      "search.exclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/*.code-search": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      }
     }
   }
 }
diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json
@@ -10,9 +10,11 @@
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
+    "./features/src/utils": {},
     "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
+    "./features/src/utils",
     "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
@@ -42,9 +44,30 @@
   "customizations": {
     "vscode": {
       "extensions": [
+        "augustocdias.tasks-shell-input",
         "ms-python.flake8",
         "nvidia.nsight-vscode-edition"
-      ]
+      ],
+      "files.watcherExclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/target/**": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      },
+      "search.exclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/*.code-search": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      }
     }
   }
 }
diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json
@@ -12,11 +12,13 @@
   "features": {
     "./features/src/ucx": {"version": "1.14.1"},
     "./features/src/cuda": {"version": "12.0", "installcuBLAS": true, "installcuDNN": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true},
+    "./features/src/utils": {},
     "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
     "./features/src/ucx",
     "./features/src/cuda",
+    "./features/src/utils",
     "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
@@ -45,9 +47,30 @@
   "customizations": {
     "vscode": {
       "extensions": [
+        "augustocdias.tasks-shell-input",
         "ms-python.flake8",
         "nvidia.nsight-vscode-edition"
-      ]
+      ],
+      "files.watcherExclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/target/**": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      },
+      "search.exclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/*.code-search": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      }
     }
   }
 }
diff --git a/.devcontainer/cuda12.2-conda/devcontainer.json b/.devcontainer/cuda12.2-conda/devcontainer.json
@@ -10,9 +10,11 @@
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
+    "./features/src/utils": {},
     "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
+    "./features/src/utils",
     "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.2-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
@@ -42,9 +44,30 @@
   "customizations": {
     "vscode": {
       "extensions": [
+        "augustocdias.tasks-shell-input",
         "ms-python.flake8",
         "nvidia.nsight-vscode-edition"
-      ]
+      ],
+      "files.watcherExclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/target/**": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      },
+      "search.exclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/*.code-search": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      }
     }
   }
 }
diff --git a/.devcontainer/cuda12.2-pip/devcontainer.json b/.devcontainer/cuda12.2-pip/devcontainer.json
@@ -12,11 +12,13 @@
   "features": {
     "./features/src/ucx": {"version": "1.14.1"},
     "./features/src/cuda": {"version": "12.2", "installcuBLAS": true, "installcuDNN": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true},
+    "./features/src/utils": {},
     "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
     "./features/src/ucx",
     "./features/src/cuda",
+    "./features/src/utils",
     "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
@@ -45,9 +47,30 @@
   "customizations": {
     "vscode": {
       "extensions": [
+        "augustocdias.tasks-shell-input",
         "ms-python.flake8",
         "nvidia.nsight-vscode-edition"
-      ]
+      ],
+      "files.watcherExclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/target/**": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      },
+      "search.exclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/*.code-search": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      }
     }
   }
 }
diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile
@@ -50,4 +50,5 @@ ENV SCCACHE_BUCKET="rapids-sccache-devs"
 ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai"
 ENV HISTFILE="/home/coder/.cache/._bash_history"
 
-ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/latest/jitify_cache"
+ENV CPM_SOURCE_CACHE="/home/coder/.cache/cpm"
+ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache"
diff --git a/.github/actions/build-linux-image/action.yml b/.github/actions/build-linux-image/action.yml
@@ -3,18 +3,9 @@ name: build-image
 description: Build Linux image
 
 inputs:
-  tag:
-    type: string
-    required: true
-    description: Image tag
-  arch:
-    type: string
-    required: true
-    description: Image arch
-  repo:
-    type: string
-    required: true
-    description: Image repo
+  tag: {type: string, required: true, description: Image tag}
+  arch: {type: string, required: true, description: Image arch}
+  repo: {type: string, required: true, description: Image repo}
 
 runs:
   using: composite

diff --git a/.github/actions/devcontainer-json/action.yml b/.github/actions/devcontainer-json/action.yml
@@ -21,7 +21,7 @@ runs:
   using: composite
   steps:
 
-    - if: env.RUNNER_ENVIRONMENT != 'github-hosted'
+    - if: runner.environment == 'self-hosted'
       name: Setup Node.js
       uses: actions/setup-node@v3
       with:

diff --git a/.github/actions/setup-runner-env/action.yml b/.github/actions/setup-runner-env/action.yml
@@ -5,45 +5,47 @@ description: Setup self-hosted runner environment
 runs:
   using: composite
   steps:
-    - name: Dump env
+    - name: Dump environment
       shell: bash -eo pipefail {0}
-      run: echo "${{ toJSON(env) }}"
+      run: |
+        echo "env: ${{ toJSON(env) }}"
+        echo "runner: '${{ toJSON(runner) }}'"
 
-    - if: env.RUNNER_ENVIRONMENT != 'self-hosted'
+    - if: runner.environment != 'self-hosted'
       name: Free up disk space
       uses: ./.github/actions/free-disk-space
       with:
         tool_cache: "${{ runner.tool_cache }}"
 
-    - if: env.RUNNER_ENVIRONMENT == 'self-hosted'
+    - if: runner.environment == 'self-hosted'
       name: Setup self-hosted runner environment
       shell: bash -eo pipefail {0}
       run: |
         echo "HOME=${{ runner.workspace }}" >> $GITHUB_ENV;
         echo "TMPDIR=${{ runner.temp }}" >> $GITHUB_ENV;
 
-    - if: env.RUNNER_ENVIRONMENT == 'self-hosted'
+    - if: runner.environment == 'self-hosted'
       name: Setup Node.js
       uses: actions/setup-node@v3
       with:
         node-version: '16'
 
-    - if: env.RUNNER_ENVIRONMENT != 'self-hosted'
+    - if: runner.environment != 'self-hosted'
       name: Set up QEMU
       uses: docker/setup-qemu-action@v2
 
     - name: Create docker context
       shell: bash
       run: docker context create builder
 
-    - if: env.RUNNER_ENVIRONMENT != 'self-hosted'
+    - if: runner.environment != 'self-hosted'
       name: Setup docker buildx on github-hosted runners
       uses: docker/setup-buildx-action@v2
       with:
         buildkitd-flags: --debug
         endpoint: builder
 
-    - if: env.RUNNER_ENVIRONMENT == 'self-hosted'
+    - if: runner.environment == 'self-hosted'
       name: Setup docker buildx on self-hosted runners
       uses: docker/setup-buildx-action@v2
       with:

diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml
@@ -37,17 +37,14 @@ jobs:
       build_command: |
         sccache -z;
         sccache --show-adv-stats;
-        clone-all --verbose --clone-upstream --no-update-env;
-        rapids-generate-scripts;
-        rapids-make-${PYTHON_PACKAGE_MANAGER}-env;
+        clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules;
         build-all \
-          --verbose \
+          -v \
           -j$(nproc --ignore=1) \
-          -DBUILD_SHARED_LIBS=ON \
           -DBUILD_TESTS=ON \
           -DBUILD_BENCHMARKS=ON \
           -DBUILD_ANN_BENCH=ON \
           -DBUILD_PRIMS_BENCH=ON \
-          $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \
+          -DBUILD_SHARED_LIBS=ON \
           ;
         sccache --show-adv-stats;