From b6fb6b1356de1830bafd9b0bde3b2e1a4c86db84 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Tue, 4 Feb 2025 17:25:32 -0700 Subject: [PATCH 01/24] added tools for Github artifact download --- tools/_rapids-download-from-github | 21 +++++++++++++ tools/rapids-download-conda-from-github | 31 ++++++++++++++++++++ tools/rapids-download-from-github | 19 ++++++++++++ tools/rapids-github-run-id | 39 +++++++++++++++++++++++++ 4 files changed, 110 insertions(+) create mode 100644 tools/_rapids-download-from-github create mode 100644 tools/rapids-download-conda-from-github create mode 100644 tools/rapids-download-from-github create mode 100644 tools/rapids-github-run-id diff --git a/tools/_rapids-download-from-github b/tools/_rapids-download-from-github new file mode 100644 index 0000000..ec671af --- /dev/null +++ b/tools/_rapids-download-from-github @@ -0,0 +1,21 @@ +#!/bin/bash +# A utility script that downloads an artifact archive from Github Artifacts, unzips it, +# and prints the location where it was unzipped. +# The script can be used locally and in CI. +# Positional Arguments: +# 1) run_id of the Github Workflow run +# 2) name of the artifact to download +# 3) location to unzip it to +set -euo pipefail +source rapids-constants + +github_run_id="$1" +pkg_name="$2" +unzip_dest="$3" + +mkdir -p "${unzip_dest}" + +rapids-echo-stderr "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" +gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${pkg_name}" --dir "${unzip_dest}" + +echo -n "${unzip_dest}" \ No newline at end of file diff --git a/tools/rapids-download-conda-from-github b/tools/rapids-download-conda-from-github new file mode 100644 index 0000000..251aed0 --- /dev/null +++ b/tools/rapids-download-conda-from-github @@ -0,0 +1,31 @@ +#!/bin/bash +# A utility script that downloads a conda artifact archive from Github Artifacts, unzips it, +# and prints the location where it was unzipped. +# Positional Arguments: +# 1) a string of "cpp" or "python" which determines which conda artifact +# should be downloaded +set -euo pipefail +export RAPIDS_SCRIPT_NAME="rapids-download-conda-from-github" + +pkg_type="$1" +case "${pkg_type}" in + cpp) + ;& + python) + ;; + *) + rapids-echo-stderr 'Pass "cpp" or "python" as an argument.' + exit 1 + ;; +esac + +source rapids-prompt-local-repo-config + +# Prepare empty dir to extract tarball to +channel="/tmp/${pkg_type}_channel" + +# Prepend `conda_` to PKG_TYPE for rapids-package-name function +pkg_type="conda_${pkg_type}" +pkg_name="$(rapids-package-name "$pkg_type")" + +rapids-download-from-github "${pkg_name}" "${channel}" \ No newline at end of file diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github new file mode 100644 index 0000000..e7d1f53 --- /dev/null +++ b/tools/rapids-download-from-github @@ -0,0 +1,19 @@ +#!/bin/bash +# A utility script that downloads an artifact from Github Artifacts, unzips it, +# and prints the location where it was unzipped. +# Positional Arguments: +# 1) package name to generate s3 path for +# 2) location to unzip it to +set -euo pipefail +export RAPIDS_SCRIPT_NAME="rapids-download-from-github" + +if [ -z "$1" ] || [ -z "$2" ]; then + rapids-echo-stderr "Must specify input arguments: PKG_NAME and UNZIP_DEST" + exit 1 +fi + +github_run_id="$(rapids-github-run-id)" +pkg_name=$(basename "$1" .tar.gz) +unzip_dest="$2" + +_rapids-download-from-github "$github_run_id" "$pkg_name" "$unzip_dest" \ No newline at end of file diff --git a/tools/rapids-github-run-id b/tools/rapids-github-run-id new file mode 100644 index 0000000..a02bae4 --- /dev/null +++ b/tools/rapids-github-run-id @@ -0,0 +1,39 @@ +#!/bin/bash +# A utility script that examines environment variables provided +# by GitHub Actions to print out a Github Workflow Run ID where the expected artifact +# should be. +# +# The output format should be one of the following: +# +## For PR builds: +## s3://rapids-downloads/ci//pull-request/// + +## For branch builds: +## s3://rapids-downloads/ci//branch/// + +## For nightly builds: +## s3://rapids-downloads/nightly//// +set -euo pipefail +source rapids-constants +export RAPIDS_SCRIPT_NAME="rapids-github-run-id" + +pkg_name=$(basename "$1" .tar.gz) + +case "${RAPIDS_BUILD_TYPE}" in + pull-request) + run_id=$(gh run list --repo ${RAPIDS_REPOSITORY} --branch ${RAPIDS_REF_NAME} --commit ${RAPIDS_SHA} --json databaseId --jq '.[0] | .databaseId') + ;; + branch) + run_id=$(gh run list --repo ${RAPIDS_REPOSITORY} --branch ${RAPIDS_REF_NAME} --commit ${RAPIDS_SHA} --json databaseId --jq '.[0] | .databaseId') + ;; + nightly) + # Need to implement this + run_id=$(gh run list --repo ${RAPIDS_REPOSITORY} --branch ${RAPIDS_REF_NAME} --commit ${RAPIDS_SHA} --json databaseId --jq '.[0] | .databaseId') + ;; + *) + rapids-echo-stderr "please pass a valid RAPIDS_BUILD_TYPE" + exit 1 + ;; +esac + +echo -n "${run_id}" \ No newline at end of file From 5b325b4befecacdffbe9238147123a99a83f8e1e Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 5 Feb 2025 16:18:51 -0700 Subject: [PATCH 02/24] made changes to download scripts --- tools/rapids-download-conda-from-github | 4 ++-- tools/rapids-download-from-github | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/rapids-download-conda-from-github b/tools/rapids-download-conda-from-github index 251aed0..5592a21 100644 --- a/tools/rapids-download-conda-from-github +++ b/tools/rapids-download-conda-from-github @@ -22,10 +22,10 @@ esac source rapids-prompt-local-repo-config # Prepare empty dir to extract tarball to -channel="/tmp/${pkg_type}_channel" +# channel="/tmp/${pkg_type}_channel" # Prepend `conda_` to PKG_TYPE for rapids-package-name function pkg_type="conda_${pkg_type}" pkg_name="$(rapids-package-name "$pkg_type")" -rapids-download-from-github "${pkg_name}" "${channel}" \ No newline at end of file +rapids-download-from-github "${pkg_name}" \ No newline at end of file diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github index e7d1f53..0552e50 100644 --- a/tools/rapids-download-from-github +++ b/tools/rapids-download-from-github @@ -1,19 +1,21 @@ #!/bin/bash -# A utility script that downloads an artifact from Github Artifacts, unzips it, +# A utility script that downloads an artifact from Github Artifacts, unzips it to a temporary directory, # and prints the location where it was unzipped. # Positional Arguments: # 1) package name to generate s3 path for -# 2) location to unzip it to set -euo pipefail export RAPIDS_SCRIPT_NAME="rapids-download-from-github" -if [ -z "$1" ] || [ -z "$2" ]; then - rapids-echo-stderr "Must specify input arguments: PKG_NAME and UNZIP_DEST" +if [ -z "$1" ] then + rapids-echo-stderr "Must specify input arguments: PKG_NAME" exit 1 fi github_run_id="$(rapids-github-run-id)" pkg_name=$(basename "$1" .tar.gz) -unzip_dest="$2" +unzip_dest="$(mktemp -d)" -_rapids-download-from-github "$github_run_id" "$pkg_name" "$unzip_dest" \ No newline at end of file +rapids-echo-stderr "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" +gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${pkg_name}" --dir "${unzip_dest}" + +echo -n "${unzip_dest}" \ No newline at end of file From e2aa648de2f74c14e880feee2b268e78e4aaaebc Mon Sep 17 00:00:00 2001 From: Jaya Venkatesh Date: Wed, 5 Feb 2025 17:18:44 -0700 Subject: [PATCH 03/24] Update release.yaml to generate releases on fork --- .github/workflows/release.yaml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4cb5f18..b43603c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,7 +3,7 @@ name: release main on: push: branches: - - main + - test-github-artifacts paths: - "tools/*" - ".github/workflows/release.yaml" @@ -35,16 +35,3 @@ jobs: with: files: ${{ env.RELEASE_FILE_NAME }} tag_name: ${{ env.NEXT_PATCH_VERSION }} - trigger-pipeline: - runs-on: ubuntu-latest - needs: release - steps: - - name: Trigger CI Images - env: - GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} - run: | - gh workflow run push.yaml \ - --field upstream_job="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - --field upstream_repository=gha-tools \ - --repo rapidsai/ci-imgs \ - --ref main From 9a86d124ba3301fb37c0aad3828f05ea9a4446f6 Mon Sep 17 00:00:00 2001 From: Jaya Venkatesh Date: Wed, 5 Feb 2025 17:20:28 -0700 Subject: [PATCH 04/24] Update release.yaml to run release action --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b43603c..8a60359 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,3 +35,4 @@ jobs: with: files: ${{ env.RELEASE_FILE_NAME }} tag_name: ${{ env.NEXT_PATCH_VERSION }} + From c40dbe58ff879a3113003cf6a733706fcddf684c Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 5 Feb 2025 17:39:51 -0700 Subject: [PATCH 05/24] made files executable --- tools/_rapids-download-from-github | 0 tools/rapids-download-conda-from-github | 0 tools/rapids-download-from-github | 0 tools/rapids-github-run-id | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/_rapids-download-from-github mode change 100644 => 100755 tools/rapids-download-conda-from-github mode change 100644 => 100755 tools/rapids-download-from-github mode change 100644 => 100755 tools/rapids-github-run-id diff --git a/tools/_rapids-download-from-github b/tools/_rapids-download-from-github old mode 100644 new mode 100755 diff --git a/tools/rapids-download-conda-from-github b/tools/rapids-download-conda-from-github old mode 100644 new mode 100755 diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github old mode 100644 new mode 100755 diff --git a/tools/rapids-github-run-id b/tools/rapids-github-run-id old mode 100644 new mode 100755 From e2d6ebdb0bbfa5ed24d7c5e81c57b1070d4ce7cf Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 5 Feb 2025 18:01:54 -0700 Subject: [PATCH 06/24] fixed error in download script --- tools/rapids-download-from-github | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github index 0552e50..ee41411 100755 --- a/tools/rapids-download-from-github +++ b/tools/rapids-download-from-github @@ -6,7 +6,7 @@ set -euo pipefail export RAPIDS_SCRIPT_NAME="rapids-download-from-github" -if [ -z "$1" ] then +if [ -z "$1" ]; then rapids-echo-stderr "Must specify input arguments: PKG_NAME" exit 1 fi From a745ffb1f3839b64d6353b31d392cdc2e01016d0 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 5 Feb 2025 18:16:50 -0700 Subject: [PATCH 07/24] fixed error in run-id script --- tools/rapids-github-run-id | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/rapids-github-run-id b/tools/rapids-github-run-id index a02bae4..d174bc8 100755 --- a/tools/rapids-github-run-id +++ b/tools/rapids-github-run-id @@ -17,8 +17,6 @@ set -euo pipefail source rapids-constants export RAPIDS_SCRIPT_NAME="rapids-github-run-id" -pkg_name=$(basename "$1" .tar.gz) - case "${RAPIDS_BUILD_TYPE}" in pull-request) run_id=$(gh run list --repo ${RAPIDS_REPOSITORY} --branch ${RAPIDS_REF_NAME} --commit ${RAPIDS_SHA} --json databaseId --jq '.[0] | .databaseId') From 3006ba4b99979d58df1772cb43eace2f8a58167d Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Mon, 10 Feb 2025 17:09:05 -0700 Subject: [PATCH 08/24] added wheel download script --- tools/rapids-download-from-github | 2 +- tools/rapids-download-wheels-from-github | 19 +++++++++++++++++++ tools/rapids-package-name | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100755 tools/rapids-download-wheels-from-github diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github index ee41411..a7e404a 100755 --- a/tools/rapids-download-from-github +++ b/tools/rapids-download-from-github @@ -12,7 +12,7 @@ if [ -z "$1" ]; then fi github_run_id="$(rapids-github-run-id)" -pkg_name=$(basename "$1" .tar.gz) +pkg_name="$1" unzip_dest="$(mktemp -d)" rapids-echo-stderr "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" diff --git a/tools/rapids-download-wheels-from-github b/tools/rapids-download-wheels-from-github new file mode 100755 index 0000000..28e4e16 --- /dev/null +++ b/tools/rapids-download-wheels-from-github @@ -0,0 +1,19 @@ +#!/bin/bash +# A utility script that downloads wheel artifacts from Github Artifacts to /wheelhouse +# Positional Arguments: +# 1) dest dir to download and extract wheels to +set -eo pipefail + +source rapids-prompt-local-repo-config + +# For legacy reasons, allow this script to be run without the pkg_type being the first arg +pkg_name="$(rapids-package-name "wheel_python")" + +pkg_type="$1" +if [ "${pkg_type}" = "cpp" ] || [ "${pkg_type}" = "python" ]; then + # remove pkg_type from args because we handle it in this script + shift; + pkg_name="$(rapids-package-name "wheel_${pkg_type}")" +fi + +rapids-download-from-github "${pkg_name}" \ No newline at end of file diff --git a/tools/rapids-package-name b/tools/rapids-package-name index eaadc5c..c076fa4 100755 --- a/tools/rapids-package-name +++ b/tools/rapids-package-name @@ -76,6 +76,6 @@ if (( append_arch == 1)); then fi # for cpp and python package types, it's a tarball, append .tar.gz and prepend project name -pkg_name="${repo_name}_${pkg_name}.tar.gz" +pkg_name="${repo_name}_${pkg_name}" echo -n "${pkg_name}" From 16ef8c0f4c21ab06cc0a8bb8fa3557ccc8143f27 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 19 Feb 2025 11:59:17 -0700 Subject: [PATCH 09/24] add newlines at the end --- tools/_rapids-download-from-github | 21 --------------------- tools/rapids-download-conda-from-github | 2 +- tools/rapids-download-from-github | 2 +- tools/rapids-download-wheels-from-github | 2 +- tools/rapids-github-run-id | 2 +- 5 files changed, 4 insertions(+), 25 deletions(-) delete mode 100755 tools/_rapids-download-from-github diff --git a/tools/_rapids-download-from-github b/tools/_rapids-download-from-github deleted file mode 100755 index ec671af..0000000 --- a/tools/_rapids-download-from-github +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# A utility script that downloads an artifact archive from Github Artifacts, unzips it, -# and prints the location where it was unzipped. -# The script can be used locally and in CI. -# Positional Arguments: -# 1) run_id of the Github Workflow run -# 2) name of the artifact to download -# 3) location to unzip it to -set -euo pipefail -source rapids-constants - -github_run_id="$1" -pkg_name="$2" -unzip_dest="$3" - -mkdir -p "${unzip_dest}" - -rapids-echo-stderr "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" -gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${pkg_name}" --dir "${unzip_dest}" - -echo -n "${unzip_dest}" \ No newline at end of file diff --git a/tools/rapids-download-conda-from-github b/tools/rapids-download-conda-from-github index 5592a21..adf62cc 100755 --- a/tools/rapids-download-conda-from-github +++ b/tools/rapids-download-conda-from-github @@ -28,4 +28,4 @@ source rapids-prompt-local-repo-config pkg_type="conda_${pkg_type}" pkg_name="$(rapids-package-name "$pkg_type")" -rapids-download-from-github "${pkg_name}" \ No newline at end of file +rapids-download-from-github "${pkg_name}" diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github index a7e404a..6ab151b 100755 --- a/tools/rapids-download-from-github +++ b/tools/rapids-download-from-github @@ -18,4 +18,4 @@ unzip_dest="$(mktemp -d)" rapids-echo-stderr "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${pkg_name}" --dir "${unzip_dest}" -echo -n "${unzip_dest}" \ No newline at end of file +echo -n "${unzip_dest}" diff --git a/tools/rapids-download-wheels-from-github b/tools/rapids-download-wheels-from-github index 28e4e16..79ee694 100755 --- a/tools/rapids-download-wheels-from-github +++ b/tools/rapids-download-wheels-from-github @@ -16,4 +16,4 @@ if [ "${pkg_type}" = "cpp" ] || [ "${pkg_type}" = "python" ]; then pkg_name="$(rapids-package-name "wheel_${pkg_type}")" fi -rapids-download-from-github "${pkg_name}" \ No newline at end of file +rapids-download-from-github "${pkg_name}" diff --git a/tools/rapids-github-run-id b/tools/rapids-github-run-id index d174bc8..7b2f2d0 100755 --- a/tools/rapids-github-run-id +++ b/tools/rapids-github-run-id @@ -34,4 +34,4 @@ case "${RAPIDS_BUILD_TYPE}" in ;; esac -echo -n "${run_id}" \ No newline at end of file +echo -n "${run_id}" From 5cf9875509c78025fe7ec36abee7f5f06d197524 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Fri, 28 Feb 2025 14:38:47 -0700 Subject: [PATCH 10/24] added support for PR artifact downloads from Github --- tools/rapids-get-pr-conda-artifact-github | 46 ++++++++++++++++++++ tools/rapids-get-pr-wheel-artifact-github | 53 +++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 tools/rapids-get-pr-conda-artifact-github create mode 100644 tools/rapids-get-pr-wheel-artifact-github diff --git a/tools/rapids-get-pr-conda-artifact-github b/tools/rapids-get-pr-conda-artifact-github new file mode 100644 index 0000000..1fc98c8 --- /dev/null +++ b/tools/rapids-get-pr-conda-artifact-github @@ -0,0 +1,46 @@ +#!/bin/bash +# Echo path to an artifact for a specific PR from GitHub Artifacts. +# Finds and uses the latest commit on the PR. +# +# Positional Arguments: +# 1) repo name +# 2) PR number +# 3) "cpp" or "python", to get the artifact for the C++ or Python build, respectively +# 4) [optional] commit hash, to get the artifact for a specific commit +# +# Example Usage: +# rapids-get-pr-conda-artifact-github rmm 1095 cpp +set -euo pipefail + +repo="${1}" +pr="${2}" +package_type="${3}" +commit="${4:-}" + +# Create a temporary directory for the artifact +unzip_dest=$(mktemp -d) + +# If commit is not provided, get the latest commit on the PR +if [[ -z "${commit}" ]]; then + commit=$(git ls-remote https://github.com/rapidsai/"${repo}".git refs/heads/pull-request/"${pr}" | cut -c1-7) +fi + +# Use a subshell to isolate environment variable changes +( + # Set up environment variables needed for GitHub artifact download + export RAPIDS_REPOSITORY="rapidsai/${repo}" + export RAPIDS_BUILD_TYPE="pull-request" + export RAPIDS_SHA="${commit}" + export RAPIDS_REF_NAME="pull-request/${pr}" + + # Generate the artifact name + artifact_name=$(rapids-package-name "conda_${package_type}") + + # Get the GitHub run ID for the PR + github_run_id=$(rapids-github-run-id) + + rapids-echo-stderr "Downloading and decompressing ${artifact_name} from GitHub Run ID ${github_run_id} into ${unzip_dest}" + gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${artifact_name}" --dir "${unzip_dest}" +) + +echo -n "${unzip_dest}" \ No newline at end of file diff --git a/tools/rapids-get-pr-wheel-artifact-github b/tools/rapids-get-pr-wheel-artifact-github new file mode 100644 index 0000000..fa6c07d --- /dev/null +++ b/tools/rapids-get-pr-wheel-artifact-github @@ -0,0 +1,53 @@ +#!/bin/bash +# Echo path to an artifact for a specific PR from GitHub Artifacts. +# Finds and uses the latest commit on the PR. +# +# Positional Arguments: +# 1) repo name +# 2) PR number +# 3) "cpp" or "python", to get the artifact for the C++ or Python build, respectively +# 4) [optional] commit hash, to get the artifact for a specific commit +# +# RAPIDS_PY_WHEEL_NAME must be set to indicate the package name to fetch. +# Example Usage: +# rapids-get-pr-wheel-artifact-github rmm 1095 cpp +set -euo pipefail + +repo="${1}" +pr="${2}" +package_type="${3}" +commit="${4:-}" + +# Check if RAPIDS_PY_WHEEL_NAME is set when fetching a wheel +if [[ -z "${RAPIDS_PY_WHEEL_NAME:+placeholder}" ]]; then + echo "The RAPIDS_PY_WHEEL_NAME variable must be set when fetching a wheel artifact" + exit 1 +fi + +# Create a temporary directory for the artifact +unzip_dest=$(mktemp -d) + +# If commit is not provided, get the latest commit on the PR +if [[ -z "${commit}" ]]; then + commit=$(git ls-remote https://github.com/rapidsai/"${repo}".git refs/heads/pull-request/"${pr}" | cut -c1-7) +fi + +# Use a subshell to isolate environment variable changes +( + # Set up environment variables needed for GitHub artifact download + export RAPIDS_REPOSITORY="rapidsai/${repo}" + export RAPIDS_BUILD_TYPE="pull-request" + export RAPIDS_SHA="${commit}" + export RAPIDS_REF_NAME="pull-request/${pr}" + + # Generate the artifact name + artifact_name=$(rapids-package-name "wheel_${package_type}") + + # Get the GitHub run ID for the PR + github_run_id=$(rapids-github-run-id) + + rapids-echo-stderr "Downloading and decompressing ${artifact_name} from GitHub Run ID ${github_run_id} into ${unzip_dest}" + gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${artifact_name}" --dir "${unzip_dest}" +) + +echo -n "${unzip_dest}" \ No newline at end of file From 92b5e117f0e9efa874aff31a6c3a55694b29af11 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Tue, 11 Mar 2025 11:36:35 -0700 Subject: [PATCH 11/24] made changes to comments and revert package name to use .tar.gz --- tools/rapids-download-conda-from-github | 2 +- tools/rapids-download-wheels-from-github | 4 +--- tools/rapids-get-pr-conda-artifact-github | 2 +- tools/rapids-get-pr-wheel-artifact-github | 2 +- tools/rapids-github-run-id | 11 ----------- tools/rapids-package-name | 2 +- 6 files changed, 5 insertions(+), 18 deletions(-) diff --git a/tools/rapids-download-conda-from-github b/tools/rapids-download-conda-from-github index adf62cc..1134f72 100755 --- a/tools/rapids-download-conda-from-github +++ b/tools/rapids-download-conda-from-github @@ -1,6 +1,6 @@ #!/bin/bash # A utility script that downloads a conda artifact archive from Github Artifacts, unzips it, -# and prints the location where it was unzipped. +# and returns the path to the directory where it was unzipped. # Positional Arguments: # 1) a string of "cpp" or "python" which determines which conda artifact # should be downloaded diff --git a/tools/rapids-download-wheels-from-github b/tools/rapids-download-wheels-from-github index 79ee694..de78e61 100755 --- a/tools/rapids-download-wheels-from-github +++ b/tools/rapids-download-wheels-from-github @@ -1,7 +1,5 @@ #!/bin/bash -# A utility script that downloads wheel artifacts from Github Artifacts to /wheelhouse -# Positional Arguments: -# 1) dest dir to download and extract wheels to +# A utility script that downloads wheel artifacts from Github Artifacts to a temporary directory and returns the path to the directory set -eo pipefail source rapids-prompt-local-repo-config diff --git a/tools/rapids-get-pr-conda-artifact-github b/tools/rapids-get-pr-conda-artifact-github index 1fc98c8..34ddf85 100644 --- a/tools/rapids-get-pr-conda-artifact-github +++ b/tools/rapids-get-pr-conda-artifact-github @@ -43,4 +43,4 @@ fi gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${artifact_name}" --dir "${unzip_dest}" ) -echo -n "${unzip_dest}" \ No newline at end of file +echo -n "${unzip_dest}" diff --git a/tools/rapids-get-pr-wheel-artifact-github b/tools/rapids-get-pr-wheel-artifact-github index fa6c07d..97fd135 100644 --- a/tools/rapids-get-pr-wheel-artifact-github +++ b/tools/rapids-get-pr-wheel-artifact-github @@ -50,4 +50,4 @@ fi gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${artifact_name}" --dir "${unzip_dest}" ) -echo -n "${unzip_dest}" \ No newline at end of file +echo -n "${unzip_dest}" diff --git a/tools/rapids-github-run-id b/tools/rapids-github-run-id index 7b2f2d0..2084369 100755 --- a/tools/rapids-github-run-id +++ b/tools/rapids-github-run-id @@ -2,17 +2,6 @@ # A utility script that examines environment variables provided # by GitHub Actions to print out a Github Workflow Run ID where the expected artifact # should be. -# -# The output format should be one of the following: -# -## For PR builds: -## s3://rapids-downloads/ci//pull-request/// - -## For branch builds: -## s3://rapids-downloads/ci//branch/// - -## For nightly builds: -## s3://rapids-downloads/nightly//// set -euo pipefail source rapids-constants export RAPIDS_SCRIPT_NAME="rapids-github-run-id" diff --git a/tools/rapids-package-name b/tools/rapids-package-name index c076fa4..eaadc5c 100755 --- a/tools/rapids-package-name +++ b/tools/rapids-package-name @@ -76,6 +76,6 @@ if (( append_arch == 1)); then fi # for cpp and python package types, it's a tarball, append .tar.gz and prepend project name -pkg_name="${repo_name}_${pkg_name}" +pkg_name="${repo_name}_${pkg_name}.tar.gz" echo -n "${pkg_name}" From b9202b1d28f0ad0a1b754a4ed6fe04ff743fc9c7 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Tue, 11 Mar 2025 13:06:49 -0700 Subject: [PATCH 12/24] reverted ci workflow --- .github/workflows/release.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8a60359..9563d83 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,7 +3,7 @@ name: release main on: push: branches: - - test-github-artifacts + - main paths: - "tools/*" - ".github/workflows/release.yaml" @@ -35,4 +35,16 @@ jobs: with: files: ${{ env.RELEASE_FILE_NAME }} tag_name: ${{ env.NEXT_PATCH_VERSION }} - + trigger-pipeline: + runs-on: ubuntu-latest + needs: release + steps: + - name: Trigger CI Images + env: + GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} + run: | + gh workflow run push.yaml \ + --field upstream_job="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + --field upstream_repository=gha-tools \ + --repo rapidsai/ci-imgs \ + --ref main \ No newline at end of file From 5ee86fc4db6fdd31d24f1beb2c61b9fca1978b1c Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Tue, 11 Mar 2025 13:21:14 -0700 Subject: [PATCH 13/24] added newline at the end --- .github/workflows/release.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9563d83..e038c1a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -47,4 +47,5 @@ jobs: --field upstream_job="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ --field upstream_repository=gha-tools \ --repo rapidsai/ci-imgs \ - --ref main \ No newline at end of file + --ref main + \ No newline at end of file From af2e8df1591898535e22478e0451b98c834e22c0 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Tue, 11 Mar 2025 13:23:00 -0700 Subject: [PATCH 14/24] fixed newline in release.yaml --- .github/workflows/release.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e038c1a..4cb5f18 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -48,4 +48,3 @@ jobs: --field upstream_repository=gha-tools \ --repo rapidsai/ci-imgs \ --ref main - \ No newline at end of file From cb79a118df1ab29a450e5735b57c5c6307d72bb9 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Tue, 11 Mar 2025 13:55:04 -0700 Subject: [PATCH 15/24] added double-quotes for shellchecker --- tools/rapids-github-run-id | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/rapids-github-run-id b/tools/rapids-github-run-id index 2084369..76c20d7 100755 --- a/tools/rapids-github-run-id +++ b/tools/rapids-github-run-id @@ -8,14 +8,13 @@ export RAPIDS_SCRIPT_NAME="rapids-github-run-id" case "${RAPIDS_BUILD_TYPE}" in pull-request) - run_id=$(gh run list --repo ${RAPIDS_REPOSITORY} --branch ${RAPIDS_REF_NAME} --commit ${RAPIDS_SHA} --json databaseId --jq '.[0] | .databaseId') + run_id=$(gh run list --repo "${RAPIDS_REPOSITORY}" --branch "${RAPIDS_REF_NAME}" --commit "${RAPIDS_SHA}" --json databaseId --jq '.[0] | .databaseId') ;; branch) - run_id=$(gh run list --repo ${RAPIDS_REPOSITORY} --branch ${RAPIDS_REF_NAME} --commit ${RAPIDS_SHA} --json databaseId --jq '.[0] | .databaseId') + run_id=$(gh run list --repo "${RAPIDS_REPOSITORY}" --branch "${RAPIDS_REF_NAME}" --commit "${RAPIDS_SHA}" --json databaseId --jq '.[0] | .databaseId') ;; nightly) - # Need to implement this - run_id=$(gh run list --repo ${RAPIDS_REPOSITORY} --branch ${RAPIDS_REF_NAME} --commit ${RAPIDS_SHA} --json databaseId --jq '.[0] | .databaseId') + run_id=$(gh run list --repo "${RAPIDS_REPOSITORY}" --branch "${RAPIDS_REF_NAME}" --commit "${RAPIDS_SHA}" --json databaseId --jq '.[0] | .databaseId') ;; *) rapids-echo-stderr "please pass a valid RAPIDS_BUILD_TYPE" From feb6f305e401e7d69c9e9eb6b421c111f905620d Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Tue, 11 Mar 2025 13:59:52 -0700 Subject: [PATCH 16/24] made files executable --- tools/rapids-get-pr-conda-artifact-github | 0 tools/rapids-get-pr-wheel-artifact-github | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/rapids-get-pr-conda-artifact-github mode change 100644 => 100755 tools/rapids-get-pr-wheel-artifact-github diff --git a/tools/rapids-get-pr-conda-artifact-github b/tools/rapids-get-pr-conda-artifact-github old mode 100644 new mode 100755 diff --git a/tools/rapids-get-pr-wheel-artifact-github b/tools/rapids-get-pr-wheel-artifact-github old mode 100644 new mode 100755 From d1bbfd721918a42e3e2017ea7a614359652526ed Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 12 Mar 2025 12:34:48 -0700 Subject: [PATCH 17/24] made changes as per code review --- tools/rapids-download-conda-from-github | 18 +++++------------- tools/rapids-download-from-github | 2 +- tools/rapids-download-wheels-from-github | 3 --- tools/rapids-get-pr-conda-artifact-github | 7 +++++-- tools/rapids-get-pr-wheel-artifact-github | 12 +++++++----- tools/rapids-github-run-id | 10 ++-------- 6 files changed, 20 insertions(+), 32 deletions(-) diff --git a/tools/rapids-download-conda-from-github b/tools/rapids-download-conda-from-github index 1134f72..77b9fd2 100755 --- a/tools/rapids-download-conda-from-github +++ b/tools/rapids-download-conda-from-github @@ -8,22 +8,14 @@ set -euo pipefail export RAPIDS_SCRIPT_NAME="rapids-download-conda-from-github" pkg_type="$1" -case "${pkg_type}" in - cpp) - ;& - python) - ;; - *) - rapids-echo-stderr 'Pass "cpp" or "python" as an argument.' - exit 1 - ;; -esac +# Validate package type argument +if [[ "${pkg_type}" != "cpp" && "${pkg_type}" != "python" ]]; then + rapids-echo-stderr 'Pass "cpp" or "python" as an argument.' + exit 1 +fi source rapids-prompt-local-repo-config -# Prepare empty dir to extract tarball to -# channel="/tmp/${pkg_type}_channel" - # Prepend `conda_` to PKG_TYPE for rapids-package-name function pkg_type="conda_${pkg_type}" pkg_name="$(rapids-package-name "$pkg_type")" diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github index 6ab151b..dd83b9c 100755 --- a/tools/rapids-download-from-github +++ b/tools/rapids-download-from-github @@ -2,7 +2,7 @@ # A utility script that downloads an artifact from Github Artifacts, unzips it to a temporary directory, # and prints the location where it was unzipped. # Positional Arguments: -# 1) package name to generate s3 path for +# 1) package name to download from Github Artifacts set -euo pipefail export RAPIDS_SCRIPT_NAME="rapids-download-from-github" diff --git a/tools/rapids-download-wheels-from-github b/tools/rapids-download-wheels-from-github index de78e61..92a90f0 100755 --- a/tools/rapids-download-wheels-from-github +++ b/tools/rapids-download-wheels-from-github @@ -4,9 +4,6 @@ set -eo pipefail source rapids-prompt-local-repo-config -# For legacy reasons, allow this script to be run without the pkg_type being the first arg -pkg_name="$(rapids-package-name "wheel_python")" - pkg_type="$1" if [ "${pkg_type}" = "cpp" ] || [ "${pkg_type}" = "python" ]; then # remove pkg_type from args because we handle it in this script diff --git a/tools/rapids-get-pr-conda-artifact-github b/tools/rapids-get-pr-conda-artifact-github index 34ddf85..40f850e 100755 --- a/tools/rapids-get-pr-conda-artifact-github +++ b/tools/rapids-get-pr-conda-artifact-github @@ -1,5 +1,5 @@ #!/bin/bash -# Echo path to an artifact for a specific PR from GitHub Artifacts. +# Downloads and decompresses an artifact for a specific PR from GitHub Artifacts. # Finds and uses the latest commit on the PR. # # Positional Arguments: @@ -8,6 +8,9 @@ # 3) "cpp" or "python", to get the artifact for the C++ or Python build, respectively # 4) [optional] commit hash, to get the artifact for a specific commit # +# Optional Environment Variables: +# RAPIDS_UNZIP_DIR: directory to unzip the artifact into +# # Example Usage: # rapids-get-pr-conda-artifact-github rmm 1095 cpp set -euo pipefail @@ -18,7 +21,7 @@ package_type="${3}" commit="${4:-}" # Create a temporary directory for the artifact -unzip_dest=$(mktemp -d) +unzip_dest="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" # If commit is not provided, get the latest commit on the PR if [[ -z "${commit}" ]]; then diff --git a/tools/rapids-get-pr-wheel-artifact-github b/tools/rapids-get-pr-wheel-artifact-github index 97fd135..2c62e99 100755 --- a/tools/rapids-get-pr-wheel-artifact-github +++ b/tools/rapids-get-pr-wheel-artifact-github @@ -1,5 +1,5 @@ #!/bin/bash -# Echo path to an artifact for a specific PR from GitHub Artifacts. +# Downloads and decompresses an artifact for a specific PR from GitHub Artifacts. # Finds and uses the latest commit on the PR. # # Positional Arguments: @@ -8,7 +8,9 @@ # 3) "cpp" or "python", to get the artifact for the C++ or Python build, respectively # 4) [optional] commit hash, to get the artifact for a specific commit # -# RAPIDS_PY_WHEEL_NAME must be set to indicate the package name to fetch. +# Optional Environment Variables: +# RAPIDS_UNZIP_DIR: directory to unzip the artifact into +# # Example Usage: # rapids-get-pr-wheel-artifact-github rmm 1095 cpp set -euo pipefail @@ -19,17 +21,17 @@ package_type="${3}" commit="${4:-}" # Check if RAPIDS_PY_WHEEL_NAME is set when fetching a wheel -if [[ -z "${RAPIDS_PY_WHEEL_NAME:+placeholder}" ]]; then +if [[ -z "${RAPIDS_PY_WHEEL_NAME}" ]]; then echo "The RAPIDS_PY_WHEEL_NAME variable must be set when fetching a wheel artifact" exit 1 fi # Create a temporary directory for the artifact -unzip_dest=$(mktemp -d) +unzip_dest="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" # If commit is not provided, get the latest commit on the PR if [[ -z "${commit}" ]]; then - commit=$(git ls-remote https://github.com/rapidsai/"${repo}".git refs/heads/pull-request/"${pr}" | cut -c1-7) + commit=$(gh pr view "${pr}" --repo rapidsai/"${repo}" --json headRefOid --jq '.headRefOid[0:7]') fi # Use a subshell to isolate environment variable changes diff --git a/tools/rapids-github-run-id b/tools/rapids-github-run-id index 76c20d7..cca2e76 100755 --- a/tools/rapids-github-run-id +++ b/tools/rapids-github-run-id @@ -3,17 +3,11 @@ # by GitHub Actions to print out a Github Workflow Run ID where the expected artifact # should be. set -euo pipefail -source rapids-constants export RAPIDS_SCRIPT_NAME="rapids-github-run-id" +# While called by CI, all the environment variables are set by the caller. However when run locally, these environment variables are set by rapids-prompt-local-repo-config case "${RAPIDS_BUILD_TYPE}" in - pull-request) - run_id=$(gh run list --repo "${RAPIDS_REPOSITORY}" --branch "${RAPIDS_REF_NAME}" --commit "${RAPIDS_SHA}" --json databaseId --jq '.[0] | .databaseId') - ;; - branch) - run_id=$(gh run list --repo "${RAPIDS_REPOSITORY}" --branch "${RAPIDS_REF_NAME}" --commit "${RAPIDS_SHA}" --json databaseId --jq '.[0] | .databaseId') - ;; - nightly) + pull-request|branch|nightly) run_id=$(gh run list --repo "${RAPIDS_REPOSITORY}" --branch "${RAPIDS_REF_NAME}" --commit "${RAPIDS_SHA}" --json databaseId --jq '.[0] | .databaseId') ;; *) From a2cd7a21f88c031a5efb64fea360ff56c6a9030d Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 12 Mar 2025 12:43:30 -0700 Subject: [PATCH 18/24] use ghcli to get latest commit --- tools/rapids-get-pr-conda-artifact-github | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rapids-get-pr-conda-artifact-github b/tools/rapids-get-pr-conda-artifact-github index 40f850e..060ea03 100755 --- a/tools/rapids-get-pr-conda-artifact-github +++ b/tools/rapids-get-pr-conda-artifact-github @@ -25,7 +25,7 @@ unzip_dest="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" # If commit is not provided, get the latest commit on the PR if [[ -z "${commit}" ]]; then - commit=$(git ls-remote https://github.com/rapidsai/"${repo}".git refs/heads/pull-request/"${pr}" | cut -c1-7) + commit=$(gh pr view "${pr}" --repo rapidsai/"${repo}" --json headRefOid --jq '.headRefOid[0:7]') fi # Use a subshell to isolate environment variable changes From 9458fc125b16d48066955bca7a3d508c37437aef Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Thu, 13 Mar 2025 10:55:34 -0700 Subject: [PATCH 19/24] made changes as per review comments --- tools/rapids-download-conda-from-github | 21 +++++++++++---------- tools/rapids-download-from-github | 2 +- tools/rapids-download-wheels-from-github | 20 +++++++++++++++----- tools/rapids-get-pr-conda-artifact-github | 21 +++++++-------------- tools/rapids-get-pr-wheel-artifact-github | 21 +++++++-------------- 5 files changed, 41 insertions(+), 44 deletions(-) diff --git a/tools/rapids-download-conda-from-github b/tools/rapids-download-conda-from-github index 77b9fd2..d63dbfd 100755 --- a/tools/rapids-download-conda-from-github +++ b/tools/rapids-download-conda-from-github @@ -7,17 +7,18 @@ set -euo pipefail export RAPIDS_SCRIPT_NAME="rapids-download-conda-from-github" -pkg_type="$1" -# Validate package type argument -if [[ "${pkg_type}" != "cpp" && "${pkg_type}" != "python" ]]; then - rapids-echo-stderr 'Pass "cpp" or "python" as an argument.' - exit 1 -fi - source rapids-prompt-local-repo-config -# Prepend `conda_` to PKG_TYPE for rapids-package-name function -pkg_type="conda_${pkg_type}" -pkg_name="$(rapids-package-name "$pkg_type")" +# Validate package type argument +pkg_type="$1" +case "${pkg_type}" in + cpp|python) + pkg_name="$(rapids-package-name "conda_${pkg_type}")" + ;; + *) + rapids-echo-stderr 'Pass "cpp" or "python" as an argument.' + exit 1 + ;; +esac rapids-download-from-github "${pkg_name}" diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github index dd83b9c..8af4a71 100755 --- a/tools/rapids-download-from-github +++ b/tools/rapids-download-from-github @@ -13,7 +13,7 @@ fi github_run_id="$(rapids-github-run-id)" pkg_name="$1" -unzip_dest="$(mktemp -d)" +unzip_dest="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" rapids-echo-stderr "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${pkg_name}" --dir "${unzip_dest}" diff --git a/tools/rapids-download-wheels-from-github b/tools/rapids-download-wheels-from-github index 92a90f0..69db92a 100755 --- a/tools/rapids-download-wheels-from-github +++ b/tools/rapids-download-wheels-from-github @@ -1,14 +1,24 @@ #!/bin/bash -# A utility script that downloads wheel artifacts from Github Artifacts to a temporary directory and returns the path to the directory +# A utility script that downloads a wheel artifact archive from Github Artifacts, unzips it, +# and returns the path to the directory where it was unzipped. +# Positional Arguments: +# 1) a string of "cpp" or "python" which determines which wheel artifact +# should be downloaded set -eo pipefail +export RAPIDS_SCRIPT_NAME="rapids-download-wheels-from-github" source rapids-prompt-local-repo-config +# Validate package type argument pkg_type="$1" -if [ "${pkg_type}" = "cpp" ] || [ "${pkg_type}" = "python" ]; then - # remove pkg_type from args because we handle it in this script - shift; +case "${pkg_type}" in + cpp|python) pkg_name="$(rapids-package-name "wheel_${pkg_type}")" -fi + ;; + *) + rapids-echo-stderr 'Pass "cpp" or "python" as an argument.' + exit 1 + ;; +esac rapids-download-from-github "${pkg_name}" diff --git a/tools/rapids-get-pr-conda-artifact-github b/tools/rapids-get-pr-conda-artifact-github index 060ea03..be1bb8c 100755 --- a/tools/rapids-get-pr-conda-artifact-github +++ b/tools/rapids-get-pr-conda-artifact-github @@ -1,28 +1,26 @@ #!/bin/bash -# Downloads and decompresses an artifact for a specific PR from GitHub Artifacts. +# Downloads and decompresses a conda artifact for a specific PR from GitHub Artifacts. # Finds and uses the latest commit on the PR. # # Positional Arguments: # 1) repo name # 2) PR number -# 3) "cpp" or "python", to get the artifact for the C++ or Python build, respectively -# 4) [optional] commit hash, to get the artifact for a specific commit +# 3) "cpp" or "python", to get the conda artifact for the C++ or Python build, respectively +# 4) [optional] commit hash, to get the conda artifact for a specific commit # # Optional Environment Variables: -# RAPIDS_UNZIP_DIR: directory to unzip the artifact into +# RAPIDS_UNZIP_DIR: directory to unzip the conda artifact into # # Example Usage: # rapids-get-pr-conda-artifact-github rmm 1095 cpp set -euo pipefail +export RAPIDS_SCRIPT_NAME="rapids-get-pr-conda-artifact-github" repo="${1}" pr="${2}" package_type="${3}" commit="${4:-}" -# Create a temporary directory for the artifact -unzip_dest="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" - # If commit is not provided, get the latest commit on the PR if [[ -z "${commit}" ]]; then commit=$(gh pr view "${pr}" --repo rapidsai/"${repo}" --json headRefOid --jq '.headRefOid[0:7]') @@ -39,11 +37,6 @@ fi # Generate the artifact name artifact_name=$(rapids-package-name "conda_${package_type}") - # Get the GitHub run ID for the PR - github_run_id=$(rapids-github-run-id) - - rapids-echo-stderr "Downloading and decompressing ${artifact_name} from GitHub Run ID ${github_run_id} into ${unzip_dest}" - gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${artifact_name}" --dir "${unzip_dest}" + unzip_dir=$(rapids-download-from-github "${artifact_name}") + echo -n "${unzip_dir}" ) - -echo -n "${unzip_dest}" diff --git a/tools/rapids-get-pr-wheel-artifact-github b/tools/rapids-get-pr-wheel-artifact-github index 2c62e99..8b5d75c 100755 --- a/tools/rapids-get-pr-wheel-artifact-github +++ b/tools/rapids-get-pr-wheel-artifact-github @@ -1,19 +1,20 @@ #!/bin/bash -# Downloads and decompresses an artifact for a specific PR from GitHub Artifacts. +# Downloads and decompresses a wheel artifact for a specific PR from GitHub Artifacts. # Finds and uses the latest commit on the PR. # # Positional Arguments: # 1) repo name # 2) PR number -# 3) "cpp" or "python", to get the artifact for the C++ or Python build, respectively -# 4) [optional] commit hash, to get the artifact for a specific commit +# 3) "cpp" or "python", to get the wheel artifact for the C++ or Python build, respectively +# 4) [optional] commit hash, to get the wheel artifact for a specific commit # # Optional Environment Variables: -# RAPIDS_UNZIP_DIR: directory to unzip the artifact into +# RAPIDS_UNZIP_DIR: directory to unzip the wheel artifact into # # Example Usage: # rapids-get-pr-wheel-artifact-github rmm 1095 cpp set -euo pipefail +export RAPIDS_SCRIPT_NAME="rapids-get-pr-wheel-artifact-github" repo="${1}" pr="${2}" @@ -26,9 +27,6 @@ if [[ -z "${RAPIDS_PY_WHEEL_NAME}" ]]; then exit 1 fi -# Create a temporary directory for the artifact -unzip_dest="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" - # If commit is not provided, get the latest commit on the PR if [[ -z "${commit}" ]]; then commit=$(gh pr view "${pr}" --repo rapidsai/"${repo}" --json headRefOid --jq '.headRefOid[0:7]') @@ -45,11 +43,6 @@ fi # Generate the artifact name artifact_name=$(rapids-package-name "wheel_${package_type}") - # Get the GitHub run ID for the PR - github_run_id=$(rapids-github-run-id) - - rapids-echo-stderr "Downloading and decompressing ${artifact_name} from GitHub Run ID ${github_run_id} into ${unzip_dest}" - gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${artifact_name}" --dir "${unzip_dest}" + unzip_dir=$(rapids-download-from-github "${artifact_name}") + echo -n "${unzip_dir}" ) - -echo -n "${unzip_dest}" From ac1067a13190272267d6b1bdbf9b418f01e37049 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Fri, 14 Mar 2025 10:00:30 -0700 Subject: [PATCH 20/24] added flag to remove .tar.gz extension --- tools/rapids-download-conda-from-github | 2 +- tools/rapids-download-wheels-from-github | 2 +- tools/rapids-get-pr-conda-artifact-github | 2 +- tools/rapids-get-pr-wheel-artifact-github | 2 +- tools/rapids-package-name | 7 ++++++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/rapids-download-conda-from-github b/tools/rapids-download-conda-from-github index d63dbfd..652047b 100755 --- a/tools/rapids-download-conda-from-github +++ b/tools/rapids-download-conda-from-github @@ -13,7 +13,7 @@ source rapids-prompt-local-repo-config pkg_type="$1" case "${pkg_type}" in cpp|python) - pkg_name="$(rapids-package-name "conda_${pkg_type}")" + pkg_name="$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "conda_${pkg_type}")" ;; *) rapids-echo-stderr 'Pass "cpp" or "python" as an argument.' diff --git a/tools/rapids-download-wheels-from-github b/tools/rapids-download-wheels-from-github index 69db92a..1eb8d71 100755 --- a/tools/rapids-download-wheels-from-github +++ b/tools/rapids-download-wheels-from-github @@ -13,7 +13,7 @@ source rapids-prompt-local-repo-config pkg_type="$1" case "${pkg_type}" in cpp|python) - pkg_name="$(rapids-package-name "wheel_${pkg_type}")" + pkg_name="$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "wheel_${pkg_type}")" ;; *) rapids-echo-stderr 'Pass "cpp" or "python" as an argument.' diff --git a/tools/rapids-get-pr-conda-artifact-github b/tools/rapids-get-pr-conda-artifact-github index be1bb8c..4dd389d 100755 --- a/tools/rapids-get-pr-conda-artifact-github +++ b/tools/rapids-get-pr-conda-artifact-github @@ -35,7 +35,7 @@ fi export RAPIDS_REF_NAME="pull-request/${pr}" # Generate the artifact name - artifact_name=$(rapids-package-name "conda_${package_type}") + artifact_name=$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "conda_${package_type}") unzip_dir=$(rapids-download-from-github "${artifact_name}") echo -n "${unzip_dir}" diff --git a/tools/rapids-get-pr-wheel-artifact-github b/tools/rapids-get-pr-wheel-artifact-github index 8b5d75c..1101f0f 100755 --- a/tools/rapids-get-pr-wheel-artifact-github +++ b/tools/rapids-get-pr-wheel-artifact-github @@ -41,7 +41,7 @@ fi export RAPIDS_REF_NAME="pull-request/${pr}" # Generate the artifact name - artifact_name=$(rapids-package-name "wheel_${package_type}") + artifact_name=$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "wheel_${package_type}") unzip_dir=$(rapids-download-from-github "${artifact_name}") echo -n "${unzip_dir}" diff --git a/tools/rapids-package-name b/tools/rapids-package-name index eaadc5c..945f2d0 100755 --- a/tools/rapids-package-name +++ b/tools/rapids-package-name @@ -76,6 +76,11 @@ if (( append_arch == 1)); then fi # for cpp and python package types, it's a tarball, append .tar.gz and prepend project name -pkg_name="${repo_name}_${pkg_name}.tar.gz" +pkg_extension=".tar.gz" +if [[ "${RAPIDS_NO_PKG_EXTENSION:-}" == "true" ]]; then + pkg_extension="" +fi + +pkg_name="${repo_name}_${pkg_name}${pkg_extension}" echo -n "${pkg_name}" From d8056d7a07672d8fcaa7019c77f98895fca8a302 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Fri, 14 Mar 2025 10:10:42 -0700 Subject: [PATCH 21/24] added double quotes to prevent globbing --- tools/rapids-get-pr-conda-artifact-github | 2 +- tools/rapids-get-pr-wheel-artifact-github | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/rapids-get-pr-conda-artifact-github b/tools/rapids-get-pr-conda-artifact-github index 4dd389d..b70aa29 100755 --- a/tools/rapids-get-pr-conda-artifact-github +++ b/tools/rapids-get-pr-conda-artifact-github @@ -35,7 +35,7 @@ fi export RAPIDS_REF_NAME="pull-request/${pr}" # Generate the artifact name - artifact_name=$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "conda_${package_type}") + artifact_name="$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "conda_${package_type}")" unzip_dir=$(rapids-download-from-github "${artifact_name}") echo -n "${unzip_dir}" diff --git a/tools/rapids-get-pr-wheel-artifact-github b/tools/rapids-get-pr-wheel-artifact-github index 1101f0f..1a60a78 100755 --- a/tools/rapids-get-pr-wheel-artifact-github +++ b/tools/rapids-get-pr-wheel-artifact-github @@ -41,7 +41,7 @@ fi export RAPIDS_REF_NAME="pull-request/${pr}" # Generate the artifact name - artifact_name=$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "wheel_${package_type}") + artifact_name="$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "wheel_${package_type}")" unzip_dir=$(rapids-download-from-github "${artifact_name}") echo -n "${unzip_dir}" From b4b0fbb632a31c736f2665077b3d9e03e422a5e7 Mon Sep 17 00:00:00 2001 From: Jaya Venkatesh Date: Fri, 14 Mar 2025 11:16:51 -0700 Subject: [PATCH 22/24] Update tools/rapids-get-pr-wheel-artifact-github Co-authored-by: Gil Forsyth --- tools/rapids-get-pr-wheel-artifact-github | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rapids-get-pr-wheel-artifact-github b/tools/rapids-get-pr-wheel-artifact-github index 1a60a78..c52a4b3 100755 --- a/tools/rapids-get-pr-wheel-artifact-github +++ b/tools/rapids-get-pr-wheel-artifact-github @@ -23,7 +23,7 @@ commit="${4:-}" # Check if RAPIDS_PY_WHEEL_NAME is set when fetching a wheel if [[ -z "${RAPIDS_PY_WHEEL_NAME}" ]]; then - echo "The RAPIDS_PY_WHEEL_NAME variable must be set when fetching a wheel artifact" + rapids-echo-stderr "The RAPIDS_PY_WHEEL_NAME variable must be set when fetching a wheel artifact" exit 1 fi From 32eca28da41267a72d76807329b94a20aee2eba0 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Fri, 14 Mar 2025 11:20:40 -0700 Subject: [PATCH 23/24] added download string to rapids-logger --- tools/rapids-download-from-github | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github index 8af4a71..6e65fac 100755 --- a/tools/rapids-download-from-github +++ b/tools/rapids-download-from-github @@ -15,7 +15,7 @@ github_run_id="$(rapids-github-run-id)" pkg_name="$1" unzip_dest="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" -rapids-echo-stderr "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" +rapids-logger "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${pkg_name}" --dir "${unzip_dest}" echo -n "${unzip_dest}" From 578ad3c9dcc0c10147beb5fb3be4e0cf90490791 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Fri, 14 Mar 2025 11:40:38 -0700 Subject: [PATCH 24/24] switched logger back to stderr --- tools/rapids-download-from-github | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rapids-download-from-github b/tools/rapids-download-from-github index 6e65fac..8af4a71 100755 --- a/tools/rapids-download-from-github +++ b/tools/rapids-download-from-github @@ -15,7 +15,7 @@ github_run_id="$(rapids-github-run-id)" pkg_name="$1" unzip_dest="${RAPIDS_UNZIP_DIR:-$(mktemp -d)}" -rapids-logger "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" +rapids-echo-stderr "Downloading and decompressing ${pkg_name} from Run ID ${github_run_id} into ${unzip_dest}" gh run download "${github_run_id}" --repo "${RAPIDS_REPOSITORY}" --name "${pkg_name}" --dir "${unzip_dest}" echo -n "${unzip_dest}"