From 3f964c9533fbac4c6bf9375fa43781013547731f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 9 Jan 2024 12:28:40 -0800 Subject: [PATCH 001/217] separate make-pip-dependencies and make-conda-dependencies scripts --- .../devcontainer-feature.json | 2 +- features/src/rapids-build-utils/install.sh | 2 + .../bin/make-conda-dependencies.sh | 56 +++++++++++++ .../rapids-build-utils/bin/make-conda-env.sh | 58 +++----------- .../bin/make-pip-dependencies.sh | 79 +++++++++++++++++++ .../rapids-build-utils/bin/make-pip-env.sh | 79 +++---------------- 6 files changed, 158 insertions(+), 118 deletions(-) create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 588dde878..6adf8ce44 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "24.2.7", + "version": "24.2.8", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/install.sh b/features/src/rapids-build-utils/install.sh index 49b9fc0dd..d3b0ef70b 100644 --- a/features/src/rapids-build-utils/install.sh +++ b/features/src/rapids-build-utils/install.sh @@ -39,7 +39,9 @@ update-alternatives --install /usr/bin/rapids-pull-repositories rapi update-alternatives --install /usr/bin/rapids-push-repositories rapids-push-repositories /opt/rapids-build-utils/bin/push-repositories.sh 0; update-alternatives --install /usr/bin/rapids-generate-scripts rapids-generate-scripts /opt/rapids-build-utils/bin/generate-scripts.sh 0; update-alternatives --install /usr/bin/rapids-join-strings rapids-join-strings /opt/rapids-build-utils/bin/join-strings.sh 0; +update-alternatives --install /usr/bin/rapids-make-conda-dependencies rapids-make-conda-dependencies /opt/rapids-build-utils/bin/make-conda-dependencies.sh 0; update-alternatives --install /usr/bin/rapids-make-conda-env rapids-make-conda-env /opt/rapids-build-utils/bin/make-conda-env.sh 0; +update-alternatives --install /usr/bin/rapids-make-pip-dependencies rapids-make-pip-dependencies /opt/rapids-build-utils/bin/make-pip-dependencies.sh 0; update-alternatives --install /usr/bin/rapids-make-pip-env rapids-make-pip-env /opt/rapids-build-utils/bin/make-pip-env.sh 0; update-alternatives --install /usr/bin/rapids-make-vscode-workspace rapids-make-vscode-workspace /opt/rapids-build-utils/bin/make-vscode-workspace.sh 0; update-alternatives --install /usr/bin/rapids-parse-cmake-args rapids-parse-cmake-args /opt/rapids-build-utils/bin/parse-cmake-args.sh 0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh new file mode 100755 index 000000000..7c548e8fa --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -0,0 +1,56 @@ +#! /usr/bin/env bash + +make_conda_dependencies() { + + set -euo pipefail; + + local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; + cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; + + local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; + python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + + local conda_noinstall=(); + local conda_env_yamls=(); + + eval "$( \ + rapids-list-repos "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + local i; + + for ((i=0; i < ${repos_length:-0}; i+=1)); do + + local repo="repos_${i}"; + local repo_name="${repo}_name"; + local repo_path="${repo}_path"; + + if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then + conda_env_yamls+=("/tmp/${!repo_name}.yaml"); + conda_noinstall+=("$(rapids-python-conda-pkg-names --repo "${!repo_name}")"); + echo "Generating ${!repo_name}'s conda env yml" 1>&2; + /opt/conda/bin/rapids-dependency-file-generator \ + --file_key all \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + > /tmp/${!repo_name}.yaml; + fi + done + + if test ${#conda_env_yamls[@]} -gt 0; then + + # Generate a combined conda env yaml file. + /opt/conda/bin/conda-merge ${conda_env_yamls[@]} \ + | grep -v '^name:' \ + | grep -v -P '^(.*?)\-(.*?)rapids-(.*?)$' \ + | grep -v -P '^(.*?)\-(.*?)(\.git\@[^(main|master)])(.*?)$' \ + | grep -v -P "^(.*?)\-(.*?)($(rapids-join-strings "|" ${conda_noinstall[@]}))(=.*|>.*|<.*)?$" \ + ; + + rm ${conda_env_yamls[@]}; + fi +} + +(make_conda_dependencies "$@"); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index a9c75e003..88c3453fd 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -4,65 +4,27 @@ make_conda_env() { set -euo pipefail; + local force=; local env_name="${1}"; local env_file_name="${env_name}.yml"; + eval "$( \ + devcontainer-utils-parse-args --names ' + f|force | + ' - <<< "${@:2}" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + # Remove the current conda env if called with `-f|--force` - if echo "${@:2}" | grep -qE '(\-f|\-\-force)'; then + if test -n "${f:-${force:-}}"; then rm -rf "$HOME/.conda/envs/${env_name}" \ "$HOME/.conda/envs/${env_file_name}"; fi - local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; - cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; - - local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; - python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; - local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.conda/envs/${env_file_name})"; - local conda_noinstall=(); - local conda_env_yamls=(); - - eval "$( \ - rapids-list-repos "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - local i; - - for ((i=0; i < ${repos_length:-0}; i+=1)); do - - local repo="repos_${i}"; - local repo_name="${repo}_name"; - local repo_path="${repo}_path"; - - if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then - conda_env_yamls+=("/tmp/${!repo_name}.yaml"); - conda_noinstall+=("$(rapids-python-conda-pkg-names --repo "${!repo_name}")"); - echo "Generating ${!repo_name}'s conda env yml" 1>&2; - /opt/conda/bin/rapids-dependency-file-generator \ - --file_key all \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ - > /tmp/${!repo_name}.yaml; - fi - done - - if test ${#conda_env_yamls[@]} -gt 0; then - - # Generate a combined conda env yaml file. - /opt/conda/bin/conda-merge ${conda_env_yamls[@]} \ - | grep -v '^name:' \ - | grep -v -P '^(.*?)\-(.*?)rapids-(.*?)$' \ - | grep -v -P '^(.*?)\-(.*?)(\.git\@[^(main|master)])(.*?)$' \ - | grep -v -P "^(.*?)\-(.*?)($(rapids-join-strings "|" ${conda_noinstall[@]}))(=.*|>.*|<.*)?$" \ - > "${new_env_path}"; - - rm ${conda_env_yamls[@]}; - fi + make-conda-dependencies > "${new_env_path}"; if test -f "${new_env_path}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh new file mode 100755 index 000000000..145421c2f --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -0,0 +1,79 @@ +#! /usr/bin/env bash + +make_pip_dependencies() { + + set -euo pipefail; + + local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; + cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; + local cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")"; + + local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; + python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + + local pip_noinstall=(); + local pip_reqs_txts=(); + + eval "$( \ + rapids-list-repos "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + local i; + + for ((i=0; i < ${repos_length:-0}; i+=1)); do + + local repo="repos_${i}"; + local repo_name="${repo}_name"; + local repo_path="${repo}_path"; + + pip_noinstall+=("lib${!repo_name}" "${!repo_name}"); + + if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then + pip_reqs_txts+=("/tmp/${!repo_name}.requirements.txt"); + + echo "Generating ${!repo_name}'s requirements.txt" 1>&2; + + rapids-dependency-file-generator \ + --file_key py_build_${!repo_name} \ + --file_key py_run_${!repo_name} \ + --file_key py_test_${!repo_name} \ + --file_key py_build \ + --file_key py_run \ + --file_key py_test \ + --file_key all \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + `# --stdout` \ + | grep -v '^#' \ + | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ + | sed -E "s/cuda-python[ ]+?>=.*\"/cuda-python>=${cuda_version_major}\"/" \ + | sed -E "s/cupy-cuda[0-9]+x[ ]+?>=.*\"/cupy-cuda${cuda_version_major}x\"/g" \ + > /tmp/${!repo_name}.requirements.txt; + fi + done + + # add all python packages to the noinstall list. + for pkg in $(rapids-python-pkg-names) $(rapids-python-conda-pkg-names); do + pip_noinstall+=("${pkg}" "${pkg}-cu.*"); + if test -z "${pkg##*"-"*}"; then + pip_noinstall+=("${pkg//"-"/"_"}" "${pkg//"-"/"_"}-cu.*") + fi + if test -z "${pkg##*"_"*}"; then + pip_noinstall+=("${pkg//"_"/"-"}" "${pkg//"_"/"-"}-cu.*") + fi + done + + if test ${#pip_reqs_txts[@]} -gt 0; then + pip_noinstall=($(echo -n "${pip_noinstall[@]}" | xargs -r -n1 -d' ' | sort -su | tr '\n' ' ')); + # Generate a combined requirements.txt file + cat ${pip_reqs_txts[@]} \ + | grep -v -P "^($(rapids-join-strings "|" ${pip_noinstall[@]}))==.*$" \ + ; + + rm ${pip_reqs_txts[@]}; + fi +} + +make_pip_dependencies "$@"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 964ff0956..d1579c4db 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -4,86 +4,27 @@ make_pip_env() { set -euo pipefail; + local force=; local env_name="${1}"; local env_file_name="${env_name}.requirements.txt"; + eval "$( \ + devcontainer-utils-parse-args --names ' + f|force | + ' - <<< "${@:2}" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + # Remove the current virtual env if called with `-f|--force` - if echo "${@:2}" | grep -qE '(\-f|\-\-force)'; then + if test -n "${f:-${force:-}}"; then rm -rf "$HOME/.local/share/venvs/${env_name}" \ "$HOME/.local/share/venvs/${env_file_name}"; fi - local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; - cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; - local cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")"; - - local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; - python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; - local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.local/share/venvs/${env_file_name})"; - local pip_noinstall=(); - local pip_reqs_txts=(); - - eval "$( \ - rapids-list-repos "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - local i; - - for ((i=0; i < ${repos_length:-0}; i+=1)); do - - local repo="repos_${i}"; - local repo_name="${repo}_name"; - local repo_path="${repo}_path"; - - pip_noinstall+=("lib${!repo_name}" "${!repo_name}"); - - if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then - pip_reqs_txts+=("/tmp/${!repo_name}.requirements.txt"); - - echo "Generating ${!repo_name}'s requirements.txt" 1>&2; - - rapids-dependency-file-generator \ - --file_key py_build_${!repo_name} \ - --file_key py_run_${!repo_name} \ - --file_key py_test_${!repo_name} \ - --file_key py_build \ - --file_key py_run \ - --file_key py_test \ - --file_key all \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ - `# --stdout` \ - | grep -v '^#' \ - | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ - > /tmp/${!repo_name}.requirements.txt; - fi - done - - # add all python packages to the noinstall list. - for pkg in $(rapids-python-pkg-names) $(rapids-python-conda-pkg-names); do - pip_noinstall+=("${pkg}" "${pkg}-cu.*"); - if test -z "${pkg##*"-"*}"; then - pip_noinstall+=("${pkg//"-"/"_"}" "${pkg//"-"/"_"}-cu.*") - fi - if test -z "${pkg##*"_"*}"; then - pip_noinstall+=("${pkg//"_"/"-"}" "${pkg//"_"/"-"}-cu.*") - fi - done - - if test ${#pip_reqs_txts[@]} -gt 0; then - pip_noinstall=($(echo -n "${pip_noinstall[@]}" | xargs -r -n1 -d' ' | sort -su | tr '\n' ' ')); - # Generate a combined requirements.txt file - cat ${pip_reqs_txts[@]} \ - | grep -v -P "^($(rapids-join-strings "|" ${pip_noinstall[@]}))==.*$" \ - > "${new_env_path}"; - - rm ${pip_reqs_txts[@]}; - fi + make-pip-dependencies > "${new_env_path}"; if test -f "${new_env_path}"; then From 8f7f8c65b8a0ca6fc9b66c038bae17c6cbc8ba1f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 9 Jan 2024 12:56:09 -0800 Subject: [PATCH 002/217] fix names --- .../opt/rapids-build-utils/bin/make-conda-env.sh | 2 +- .../opt/rapids-build-utils/bin/make-pip-env.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index 88c3453fd..f6f96e696 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -24,7 +24,7 @@ make_conda_env() { local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.conda/envs/${env_file_name})"; - make-conda-dependencies > "${new_env_path}"; + rapids-make-conda-dependencies > "${new_env_path}"; if test -f "${new_env_path}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index d1579c4db..ddc026d29 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -24,7 +24,7 @@ make_pip_env() { local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.local/share/venvs/${env_file_name})"; - make-pip-dependencies > "${new_env_path}"; + rapids-make-pip-dependencies > "${new_env_path}"; if test -f "${new_env_path}"; then From 05203ca5ddc9e27e6e0f2a1bc1b6d98f658b6c4c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 10 Jan 2024 12:05:29 -0800 Subject: [PATCH 003/217] parse duplicated arguments into a list --- features/src/utils/devcontainer-feature.json | 2 +- .../utils/opt/devcontainer/bin/parse-args.sh | 26 ++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index effdbe217..f0f3fb3de 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "24.2.3", + "version": "24.2.4", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/utils/opt/devcontainer/bin/parse-args.sh b/features/src/utils/opt/devcontainer/bin/parse-args.sh index 4f0600ec3..ff260680d 100755 --- a/features/src/utils/opt/devcontainer/bin/parse-args.sh +++ b/features/src/utils/opt/devcontainer/bin/parse-args.sh @@ -87,7 +87,15 @@ parse_args() { if grep -qP "^${vars}$" <<< "${key}"; then key="${key//-/_}"; keys+=("$key"); - dict[$key]="$(printf %q "${val}")"; + if test -v dict[$key]; then + dict[$key]+=" $(printf %q "${val}")"; + if ! test -v __${key}_ary; then + declare __${key}_ary; + eval __${key}_ary=""; + fi + else + dict[$key]="$(printf %q "${val}")"; + fi else rest+=("${arg}"); fi @@ -96,17 +104,21 @@ parse_args() { keys+=("__rest__"); dict["__rest__"]="(${rest[@]})"; - keys=($(echo "${keys[@]}" | xargs -r -d' ' -n1 echo -e | sort -s | uniq)); + keys=($(echo "${keys[@]}" | xargs -r -d' ' -n1 echo -e | sort -su)); { set +x; } 2>/dev/null; - local keyi=1; + local keyi=0; - for ((keyi=1; keyi < ${#keys[@]}; keyi+=1)); do - echo "${keys[$keyi]}=${dict[${keys[$keyi]}]}"; + for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + key=${keys[$keyi]}; + val=${dict[$key]}; + if ! test -v __${key}_ary; then + echo "${key}=${val}"; + else + echo "${key}=(${val})"; + fi done - - echo "${keys[0]}=${dict[${keys[0]}]}"; } if test -n "${devcontainer_utils_debug:-}"; then From 0c6d3aa1b7351db55d53079a6740124e73375926 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 10 Jan 2024 12:07:06 -0800 Subject: [PATCH 004/217] support passing -k|--keys to rapids-make-{conda,pip}-dependencies --- .../bin/make-conda-dependencies.sh | 46 ++++++++++++---- .../rapids-build-utils/bin/make-conda-env.sh | 6 +- .../bin/make-pip-dependencies.sh | 55 ++++++++++++------- .../rapids-build-utils/bin/make-pip-env.sh | 6 +- 4 files changed, 82 insertions(+), 31 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 7c548e8fa..6911f5f8b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -4,6 +4,20 @@ make_conda_dependencies() { set -euo pipefail; + local keys=(); + + eval "$( \ + devcontainer-utils-parse-args --names ' + k|keys | + ' - <<< "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + if test -v k; then keys=(${k[@]}); fi + if test ${#keys[@]} -eq 0; then + keys=(all); + fi + local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; @@ -27,24 +41,32 @@ make_conda_dependencies() { local repo_path="${repo}_path"; if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then - conda_env_yamls+=("/tmp/${!repo_name}.yaml"); conda_noinstall+=("$(rapids-python-conda-pkg-names --repo "${!repo_name}")"); + echo "Generating ${!repo_name}'s conda env yml" 1>&2; - /opt/conda/bin/rapids-dependency-file-generator \ - --file_key all \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ - > /tmp/${!repo_name}.yaml; + + local repo_keys=(${keys[@]} ${keys[@]/%/_${!repo_name}}); + local keyi; + + for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do + if rapids-dependency-file-generator \ + --file_key ${repo_keys[$keyi]} \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + >> /tmp/${!repo_name}${keyi}.yaml 2>/dev/null; then + conda_env_yamls+=("/tmp/${!repo_name}${keyi}.yaml"); + fi + done fi done if test ${#conda_env_yamls[@]} -gt 0; then # Generate a combined conda env yaml file. - /opt/conda/bin/conda-merge ${conda_env_yamls[@]} \ - | grep -v '^name:' \ - | grep -v -P '^(.*?)\-(.*?)rapids-(.*?)$' \ + conda-merge ${conda_env_yamls[@]} \ + | grep -v '^name:' \ + | grep -v -P '^(.*?)\-(.*?)rapids-(.*?)$' \ | grep -v -P '^(.*?)\-(.*?)(\.git\@[^(main|master)])(.*?)$' \ | grep -v -P "^(.*?)\-(.*?)($(rapids-join-strings "|" ${conda_noinstall[@]}))(=.*|>.*|<.*)?$" \ ; @@ -53,4 +75,8 @@ make_conda_dependencies() { fi } +if test -n "${rapids_build_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + (make_conda_dependencies "$@"); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index f6f96e696..4d5e53aa2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -24,7 +24,7 @@ make_conda_env() { local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.conda/envs/${env_file_name})"; - rapids-make-conda-dependencies > "${new_env_path}"; + rapids-make-conda-dependencies ${__rest__[@]} > "${new_env_path}"; if test -f "${new_env_path}"; then @@ -58,6 +58,10 @@ make_conda_env() { fi } +if test -n "${rapids_build_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + . /opt/conda/etc/profile.d/conda.sh; . /opt/conda/etc/profile.d/mamba.sh; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 145421c2f..5ede462f0 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -4,6 +4,20 @@ make_pip_dependencies() { set -euo pipefail; + local keys=(); + + eval "$( \ + devcontainer-utils-parse-args --names ' + k|keys | + ' - <<< "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + if test -v k; then keys=(${k[@]}); fi + if test ${#keys[@]} -eq 0; then + keys=(py_build py_run py_test all); + fi + local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; local cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")"; @@ -34,28 +48,23 @@ make_pip_dependencies() { echo "Generating ${!repo_name}'s requirements.txt" 1>&2; - rapids-dependency-file-generator \ - --file_key py_build_${!repo_name} \ - --file_key py_run_${!repo_name} \ - --file_key py_test_${!repo_name} \ - --file_key py_build \ - --file_key py_run \ - --file_key py_test \ - --file_key all \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ - `# --stdout` \ - | grep -v '^#' \ - | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ - | sed -E "s/cuda-python[ ]+?>=.*\"/cuda-python>=${cuda_version_major}\"/" \ - | sed -E "s/cupy-cuda[0-9]+x[ ]+?>=.*\"/cupy-cuda${cuda_version_major}x\"/g" \ - > /tmp/${!repo_name}.requirements.txt; + local repo_keys=(${keys[@]} ${keys[@]/%/_${!repo_name}}); + local keyi; + + for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do + rapids-dependency-file-generator \ + --file_key ${repo_keys[$keyi]} \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + `# --stdout` \ + >> /tmp/${!repo_name}.requirements.txt 2>/dev/null || true; + done fi done # add all python packages to the noinstall list. - for pkg in $(rapids-python-pkg-names) $(rapids-python-conda-pkg-names); do + for pkg in $(rapids-python-pkg-names); do pip_noinstall+=("${pkg}" "${pkg}-cu.*"); if test -z "${pkg##*"-"*}"; then pip_noinstall+=("${pkg//"-"/"_"}" "${pkg//"-"/"_"}-cu.*") @@ -69,11 +78,19 @@ make_pip_dependencies() { pip_noinstall=($(echo -n "${pip_noinstall[@]}" | xargs -r -n1 -d' ' | sort -su | tr '\n' ' ')); # Generate a combined requirements.txt file cat ${pip_reqs_txts[@]} \ - | grep -v -P "^($(rapids-join-strings "|" ${pip_noinstall[@]}))==.*$" \ + | grep -v '^#' \ + | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ + | sed -E "s/cupy-cuda[0-9]+x[ ]+?>=.*/cupy-cuda${cuda_version_major}x>=${cuda_version_major}/g" \ + | sed -E "s/cuda-python[ ]+?>=.*/cuda-python>=${cuda_version_major},<$((cuda_version_major+1)).0a0/g" \ + | grep -v -P "^($(rapids-join-strings "|" ${pip_noinstall[@]}))" \ ; rm ${pip_reqs_txts[@]}; fi } +if test -n "${rapids_build_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + make_pip_dependencies "$@"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index ddc026d29..c71d244d4 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -24,7 +24,7 @@ make_pip_env() { local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.local/share/venvs/${env_file_name})"; - rapids-make-pip-dependencies > "${new_env_path}"; + rapids-make-pip-dependencies ${__rest__[@]} > "${new_env_path}"; if test -f "${new_env_path}"; then @@ -62,6 +62,10 @@ make_pip_env() { fi } +if test -n "${rapids_build_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + (make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@"); if test -f ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate; then From b09ac4c2902669d8238b5ef8efd4eb2f36338b1d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 11 Jan 2024 11:03:21 -0800 Subject: [PATCH 005/217] fix cugraph-service-{client,server} paths --- .../rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index ae5e652b5..405028dda 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -172,10 +172,10 @@ repos: sub_dir: python/cugraph-pyg depends: [cugraph] - name: cugraph-service-client - sub_dir: python/cugraph-service/server + sub_dir: python/cugraph-service/client depends: [] - name: cugraph-service-server - sub_dir: python/cugraph-service/client + sub_dir: python/cugraph-service/server depends: [cudf, dask-cudf, cugraph, cugraph-service-client] git: repo: cugraph From 35f65d808474b6e0266f9298b920485cdc5180b0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 11 Jan 2024 11:53:46 -0800 Subject: [PATCH 006/217] escape rest parameters --- features/src/utils/opt/devcontainer/bin/parse-args.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/features/src/utils/opt/devcontainer/bin/parse-args.sh b/features/src/utils/opt/devcontainer/bin/parse-args.sh index ff260680d..f04106a48 100755 --- a/features/src/utils/opt/devcontainer/bin/parse-args.sh +++ b/features/src/utils/opt/devcontainer/bin/parse-args.sh @@ -37,7 +37,7 @@ parse_args() { # -- elif grep -qP '^--$' <<< "${arg:-}"; then shift; - rest+=("${@}"); + rest+=("${@@Q}"); break; # -foo=bar | --foo=bar elif grep -qP '^--?[^\s]+=.*$' <<< "${arg:-}"; then @@ -80,7 +80,7 @@ parse_args() { fi fi else - rest+=("${@}"); + rest+=("${@@Q}"); break; fi @@ -88,13 +88,13 @@ parse_args() { key="${key//-/_}"; keys+=("$key"); if test -v dict[$key]; then - dict[$key]+=" $(printf %q "${val}")"; + dict[$key]+=" ${val@Q}"; if ! test -v __${key}_ary; then declare __${key}_ary; eval __${key}_ary=""; fi else - dict[$key]="$(printf %q "${val}")"; + dict[$key]="${val@Q}"; fi else rest+=("${arg}"); From c55e011fc5539056c1ef285b4103f3bddc614063 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 11 Jan 2024 11:54:14 -0800 Subject: [PATCH 007/217] support python cmake, wheel, and install args in manifest.yaml --- .../bin/generate-scripts.sh | 41 +++++----- .../bin/tmpl/python-build-editable.tmpl.sh | 47 +++++++++--- .../bin/tmpl/python-build-wheel.tmpl.sh | 74 ++++++++++++++----- 3 files changed, 112 insertions(+), 50 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index b19f79994..580b2e0e4 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -30,7 +30,10 @@ generate_script() { $GIT_TAG $GIT_REPO $GIT_HOST - $GIT_UPSTREAM' \ + $GIT_UPSTREAM + $PY_CMAKE_ARGS + $PIP_WHEEL_ARGS + $PIP_INSTALL_ARGS' \ | sudo tee "$TMP_SCRIPT_DIR/${bin}" >/dev/null; sudo chmod +x "$TMP_SCRIPT_DIR/${bin}"; @@ -133,7 +136,7 @@ generate_scripts() { local j; local k; - local repo_name_all=() + local repo_names=(); for ((i=0; i < ${repos_length:-0}; i+=1)); do @@ -149,7 +152,7 @@ generate_scripts() { repo_name="$(tr "[:upper:]" "[:lower:]" <<< "${!repo_name:-}")"; - repo_name_all+=($repo_name) + repo_names+=($repo_name) # Generate a clone script for each repo ( @@ -228,26 +231,26 @@ generate_scripts() { for ((j=0; j < ${!py_length:-0}; j+=1)); do local py_name="${repo}_python_${j}_name"; - local py_args="${repo}_python_${j}_args"; + local py_cmake_args="${repo}_python_${j}_args_cmake"; + local pip_wheel_args="${repo}_python_${j}_args_wheel"; + local pip_install_args="${repo}_python_${j}_args_install"; local py_sub_dir="${repo}_python_${j}_sub_dir"; local py_depends_length="${repo}_python_${j}_depends_length"; local py_path="${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}"; - py_libs+=(${!py_name}) - py_dirs+=($py_path) - done; + py_libs+=(${!py_name}); - for ((k=0; k < ${#py_libs[@]}; k+=1)); do - local py_dir="${py_dirs[$k]}"; - local py_lib="${py_libs[$k]}"; ( - PY_SRC="${py_dir}" \ - PY_LIB="${py_lib}" \ - CPP_ARGS="${args[@]}" \ - CPP_DEPS="${deps[@]}" \ - generate_python_scripts; + PY_SRC="${py_path}" \ + PY_LIB="${!py_name}" \ + CPP_ARGS="${args[@]}" \ + CPP_DEPS="${deps[@]}" \ + PY_CMAKE_ARGS="${!py_cmake_args:-}" \ + PIP_WHEEL_ARGS="${!pip_wheel_args:-}" \ + PIP_INSTALL_ARGS="${!pip_install_args:-}" \ + generate_python_scripts ; ) || true; - done + done; for ((k=0; k < ${#cpp_libs[@]}; k+=1)); do cpp_libs[$k]="$(tr "[:upper:]" "[:lower:]" <<< "${cpp_libs[$k]}")"; @@ -271,9 +274,9 @@ generate_scripts() { for script in "clone" "clean" "configure" "build"; do # Generate a script to run a script for all repos ( - NAMES="${repo_name_all[@]}" \ - SCRIPT="${script}" \ - generate_all_script ; + NAMES="${repo_names[@]}" \ + SCRIPT="${script}" \ + generate_all_script ; ) || true; done; } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh index c8f9d71e5..8f74c15fc 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh @@ -4,27 +4,42 @@ build_${PY_LIB}_python_inplace() { set -euo pipefail; - if [[ ! -d ~/${PY_SRC} ]]; then + if [[ ! -d ~/"${PY_SRC}" ]]; then exit 1; fi - local verbose=""; + local archs=""; local parallel=""; + local max_device_obj_memory_usage=""; + + local verbose=""; eval "$( \ devcontainer-utils-parse-args --names ' a|archs | j|parallel | - v|verbose | m|max-device-obj-memory-usage | + v|verbose | ' - <<< "$@" \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; verbose="${v:-${verbose:-}}"; - parallel="${j:-${parallel:-${JOBS:-${PARALLEL_LEVEL:-$(nproc --ignore=2)}}}}"; + config_settings="${C:-${config_settings:-}}"; + + archs="${a:-${archs:-}}"; + parallel="${j:-${parallel:-}}"; + max_device_obj_memory_usage="${m:-${max_device_obj_memory_usage:-}}"; + + eval "$( \ + rapids-get-num-archs-jobs-and-load \ + ${archs:+-a ${archs}} \ + ${parallel:+-j ${parallel}} \ + ${max_device_obj_memory_usage:+-m ${max_device_obj_memory_usage}} \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; - local cmake_args=(); + local cmake_args=(${PY_CMAKE_ARGS}); if test -n "${verbose}"; then cmake_args+=("--log-level=VERBOSE"); @@ -39,29 +54,39 @@ build_${PY_LIB}_python_inplace() { if test -n "${verbose}"; then ninja_args+=("-v"); fi - if test -n "${parallel}"; then - if [ "${parallel:-}" = "true" ]; then - parallel=""; - fi - ninja_args+=("-j${parallel}"); + + if test -n "${n_jobs}"; then + ninja_args+=("-j${n_jobs}"); + fi + + if test -n "${n_load}"; then + ninja_args+=("-l${n_load}"); fi - local pip_args=(); + local pip_args=(${PIP_INSTALL_ARGS}); + if test -n "${verbose}"; then pip_args+=("-vv"); fi + pip_args+=("--no-build-isolation"); pip_args+=("--no-deps"); pip_args+=("--editable"); pip_args+=(~/"${PY_SRC}"); + cmake_args+=("-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON"); trap "rm -rf ~/'${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; + local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; + local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; + time \ + CUDAFLAGS="${cudaflags}" \ CMAKE_GENERATOR="Ninja" \ CMAKE_ARGS="${cmake_args[@]}" \ SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ + NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ SETUPTOOLS_ENABLE_FEATURES="legacy-editable" \ python -m pip install ${pip_args[@]} \ ; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh index bf31b45c7..c1f9b8144 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh @@ -4,36 +4,52 @@ build_${PY_LIB}_python_wheel() { set -euo pipefail; - if [[ ! -d ~/${PY_SRC} ]]; then + if [[ ! -d ~/"${PY_SRC}" ]]; then exit 1; fi - local verbose=""; + local archs=""; local parallel=""; + local max_device_obj_memory_usage=""; + + local verbose=""; local wheel_dir=""; eval "$( \ devcontainer-utils-parse-args --names ' a|archs | j|parallel | - v|verbose | m|max-device-obj-memory-usage | + v|verbose | w|wheel-dir | prefer-binary | only-binary | - no-verify | - no-build-isolation | no-deps | + no-clean | + no-verify | + no-cache-dir | no-use-pep517 | + no-build-isolation | ' - <<< "$@" \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; verbose="${v:-${verbose:-}}"; wheel_dir="${w:-${wheel_dir:-}}"; - parallel="${j:-${parallel:-${JOBS:-${PARALLEL_LEVEL:-$(nproc --ignore=2)}}}}"; - local cmake_args=(); + archs="${a:-${archs:-}}"; + parallel="${j:-${parallel:-}}"; + max_device_obj_memory_usage="${m:-${max_device_obj_memory_usage:-}}"; + + eval "$( \ + rapids-get-num-archs-jobs-and-load \ + ${archs:+-a ${archs}} \ + ${parallel:+-j ${parallel}} \ + ${max_device_obj_memory_usage:+-m ${max_device_obj_memory_usage}} \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + local cmake_args=(${PY_CMAKE_ARGS}); if test -n "${verbose}"; then cmake_args+=("--log-level=VERBOSE"); @@ -45,27 +61,23 @@ build_${PY_LIB}_python_wheel() { cmake_args+=(${__rest__[@]}); local ninja_args=(); + if test -n "${verbose}"; then ninja_args+=("-v"); fi - if test -n "${parallel}"; then - if [ "${parallel:-}" = "true" ]; then - parallel=""; - fi - ninja_args+=("-j${parallel}"); - fi - local pip_args=(); - if test -n "${verbose}"; then - pip_args+=("-vv"); + if test -n "${n_jobs}"; then + ninja_args+=("-j${n_jobs}"); fi - if test -n "${no_build_isolation:-}"; then - pip_args+=("--no-build-isolation"); + if test -n "${n_load}"; then + ninja_args+=("-l${n_load}"); fi - if test -n "${no_deps:-}"; then - pip_args+=("--no-deps"); + local pip_args=(${PIP_WHEEL_ARGS}); + + if test -n "${verbose}"; then + pip_args+=("-vv"); fi if test -n "${wheel_dir:-}"; then @@ -86,22 +98,44 @@ build_${PY_LIB}_python_wheel() { fi fi + if test -n "${no_deps:-}"; then + pip_args+=("--no-deps"); + fi + + if test -n "${no_clean:-}"; then + pip_args+=("--no-clean"); + fi + if test -n "${no_verify:-}"; then pip_args+=("--no-verify"); fi + if test -n "${no_cache_dir:-}"; then + pip_args+=("--no-cache-dir"); + fi + if test -n "${no_use_pep517:-}"; then pip_args+=("--no-use-pep517"); fi + if test -n "${no_build_isolation:-}"; then + pip_args+=("--no-build-isolation"); + fi + pip_args+=(~/"${PY_SRC}"); trap "rm -rf ~/'${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; + local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; + local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; + time \ + CUDAFLAGS="${cudaflags}" \ CMAKE_GENERATOR="Ninja" \ + PARALLEL_LEVEL="${n_jobs}" \ CMAKE_ARGS="${cmake_args[@]}" \ SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ + NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ python -m pip wheel ${pip_args[@]} \ ; } From ddefcce075962140506fd3887136035205ed4f6c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 11 Jan 2024 12:25:23 -0800 Subject: [PATCH 008/217] generate dependencies per python lib + file_key --- .../bin/make-conda-dependencies.sh | 52 ++++++------ .../bin/make-pip-dependencies.sh | 80 +++++++++++-------- 2 files changed, 73 insertions(+), 59 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 6911f5f8b..edf05a2d2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -24,11 +24,10 @@ make_conda_dependencies() { local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; - local conda_noinstall=(); local conda_env_yamls=(); eval "$( \ - rapids-list-repos "$@" \ + rapids-list-repos ${__rest__[@]} \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; @@ -41,34 +40,39 @@ make_conda_dependencies() { local repo_path="${repo}_path"; if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then - conda_noinstall+=("$(rapids-python-conda-pkg-names --repo "${!repo_name}")"); - - echo "Generating ${!repo_name}'s conda env yml" 1>&2; - - local repo_keys=(${keys[@]} ${keys[@]/%/_${!repo_name}}); - local keyi; - - for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - if rapids-dependency-file-generator \ - --file_key ${repo_keys[$keyi]} \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ - >> /tmp/${!repo_name}${keyi}.yaml 2>/dev/null; then - conda_env_yamls+=("/tmp/${!repo_name}${keyi}.yaml"); - fi + + local py_length="${repo}_python_length"; + + for ((j=0; j < ${!py_length:-0}; j+=1)); do + local py_name="${repo}_python_${j}_name"; + + echo "Generating ${!py_name}'s conda env yml" 1>&2; + + local repo_keys=(${keys[@]} ${keys[@]/%/_${!py_name//"-"/"_"}}); + local keyi; + + for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do + if rapids-dependency-file-generator \ + --file_key ${repo_keys[$keyi]} \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + `# --stdout` \ + >> /tmp/${!repo_name}.${!py_name}.${keyi}.env.yaml 2>/dev/null; then + conda_env_yamls+=("/tmp/${!repo_name}.${!py_name}.${keyi}.env.yaml"); + fi + done done fi done if test ${#conda_env_yamls[@]} -gt 0; then - + local conda_noinstall=($(rapids-python-pkg-names) $(rapids-python-conda-pkg-names)); # Generate a combined conda env yaml file. - conda-merge ${conda_env_yamls[@]} \ - | grep -v '^name:' \ - | grep -v -P '^(.*?)\-(.*?)rapids-(.*?)$' \ - | grep -v -P '^(.*?)\-(.*?)(\.git\@[^(main|master)])(.*?)$' \ - | grep -v -P "^(.*?)\-(.*?)($(rapids-join-strings "|" ${conda_noinstall[@]}))(=.*|>.*|<.*)?$" \ + conda-merge ${conda_env_yamls[@]} \ + | grep -v '^name:' \ + | grep -v -P '^[ ]*?\- (\.git\@[^(main|master)])(.*?)$' \ + | grep -v -P "^[ ]*?\- ($(tr -d '[:blank:]' <<< "${conda_noinstall[@]/%/ |}"))(=.*|>.*|<.*)?$" \ ; rm ${conda_env_yamls[@]}; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 5ede462f0..dedeef342 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -19,17 +19,20 @@ make_pip_dependencies() { fi local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; - cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; + cuda_version="$(grep -o '^[0-9]*.[0-9]*.[0-9]' <<< "${cuda_version}")"; local cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")"; + local cuda_version_minor="$(cut -d'.' -f2 <<< "${cuda_version}")"; + local cuda_version_patch="$(cut -d'.' -f3 <<< "${cuda_version}")"; + local cuda_version_mm="${cuda_version_major}.${cuda_version_minor:-0}"; + cuda_version="${cuda_version_mm}.${cuda_version_patch:-0}"; local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; - local pip_noinstall=(); local pip_reqs_txts=(); eval "$( \ - rapids-list-repos "$@" \ + rapids-list-repos ${__rest__[@]} \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; @@ -41,48 +44,55 @@ make_pip_dependencies() { local repo_name="${repo}_name"; local repo_path="${repo}_path"; - pip_noinstall+=("lib${!repo_name}" "${!repo_name}"); - if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then - pip_reqs_txts+=("/tmp/${!repo_name}.requirements.txt"); - echo "Generating ${!repo_name}'s requirements.txt" 1>&2; + local py_length="${repo}_python_length"; - local repo_keys=(${keys[@]} ${keys[@]/%/_${!repo_name}}); - local keyi; + for ((j=0; j < ${!py_length:-0}; j+=1)); do + local py_name="${repo}_python_${j}_name"; - for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - rapids-dependency-file-generator \ - --file_key ${repo_keys[$keyi]} \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ - `# --stdout` \ - >> /tmp/${!repo_name}.requirements.txt 2>/dev/null || true; - done - fi - done + echo "Generating ${!py_name}'s requirements.txt" 1>&2; - # add all python packages to the noinstall list. - for pkg in $(rapids-python-pkg-names); do - pip_noinstall+=("${pkg}" "${pkg}-cu.*"); - if test -z "${pkg##*"-"*}"; then - pip_noinstall+=("${pkg//"-"/"_"}" "${pkg//"-"/"_"}-cu.*") - fi - if test -z "${pkg##*"_"*}"; then - pip_noinstall+=("${pkg//"_"/"-"}" "${pkg//"_"/"-"}-cu.*") + local repo_keys=(${keys[@]} ${keys[@]/%/_${!py_name//"-"/"_"}}); + local keyi; + + for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do + if rapids-dependency-file-generator \ + --file_key ${repo_keys[$keyi]} \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version_mm};py=${python_version}" \ + `# --stdout` \ + >> /tmp/${!repo_name}.${!py_name}.${keyi}.requirements.txt 2>/dev/null; then + pip_reqs_txts+=("/tmp/${!repo_name}.${!py_name}.${keyi}.requirements.txt"); + fi + done + done fi done if test ${#pip_reqs_txts[@]} -gt 0; then - pip_noinstall=($(echo -n "${pip_noinstall[@]}" | xargs -r -n1 -d' ' | sort -su | tr '\n' ' ')); + + local pip_noinstall=(); + + # add all python packages to the noinstall list. + for pkg in $(rapids-python-pkg-names); do + pip_noinstall+=("${pkg}" "${pkg}-cu.*"); + if test -z "${pkg##*"-"*}"; then + pip_noinstall+=("${pkg//"-"/"_"}" "${pkg//"-"/"_"}-cu.*") + fi + if test -z "${pkg##*"_"*}"; then + pip_noinstall+=("${pkg//"_"/"-"}" "${pkg//"_"/"-"}-cu.*") + fi + done + # Generate a combined requirements.txt file - cat ${pip_reqs_txts[@]} \ - | grep -v '^#' \ - | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ - | sed -E "s/cupy-cuda[0-9]+x[ ]+?>=.*/cupy-cuda${cuda_version_major}x>=${cuda_version_major}/g" \ - | sed -E "s/cuda-python[ ]+?>=.*/cuda-python>=${cuda_version_major},<$((cuda_version_major+1)).0a0/g" \ - | grep -v -P "^($(rapids-join-strings "|" ${pip_noinstall[@]}))" \ + cat ${pip_reqs_txts[@]} \ + | grep -v '^#' \ + | grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" \ + | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ + | sed -E "s/cupy-cuda[0-9]+x[ ]+?>=.*/cupy-cuda${cuda_version_major}x>=${cuda_version}/g" \ + | sed -E "s/cuda-python[ ]+?>=.*/cuda-python>=${cuda_version},<$((cuda_version_major+1)).0a0/g" \ ; rm ${pip_reqs_txts[@]}; From 6bfea463b973dbfebf063db3abe7a00a3bfb6eb9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 11 Jan 2024 12:26:02 -0800 Subject: [PATCH 009/217] add query-manifest command --- features/src/rapids-build-utils/install.sh | 1 + .../opt/rapids-build-utils/bin/list-repos.sh | 27 ++-------- .../rapids-build-utils/bin/query-manifest.sh | 49 +++++++++++++++++++ 3 files changed, 53 insertions(+), 24 deletions(-) create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/query-manifest.sh diff --git a/features/src/rapids-build-utils/install.sh b/features/src/rapids-build-utils/install.sh index d3b0ef70b..556b092ec 100644 --- a/features/src/rapids-build-utils/install.sh +++ b/features/src/rapids-build-utils/install.sh @@ -53,6 +53,7 @@ update-alternatives --install /usr/bin/rapids-python-pkg-names rapi update-alternatives --install /usr/bin/rapids-python-conda-pkg-names rapids-python-conda-pkg-names /opt/rapids-build-utils/bin/python-conda-pkg-names.sh 0; update-alternatives --install /usr/bin/rapids-get-num-archs-jobs-and-load rapids-get-num-archs-jobs-and-load /opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh 0; update-alternatives --install /usr/bin/rapids-list-repos rapids-list-repos /opt/rapids-build-utils/bin/list-repos.sh 0; +update-alternatives --install /usr/bin/rapids-query-manifest rapids-query-manifest /opt/rapids-build-utils/bin/query-manifest.sh 0; find /opt/rapids-build-utils \ \( -type d -exec chmod 0775 {} \; \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh index 0a7340aa9..2703f1113 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh @@ -4,49 +4,28 @@ list_repos() { set -euo pipefail; local repo=; - local manifest=; eval "$( \ devcontainer-utils-parse-args --names ' r|repo | - m|manifest | ' - <<< "$@" \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; repo="${r:-${repo:-}}"; - manifest="${m:-${manifest:-"${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}"}}"; - - local query=; - - query="$(cat <<"________EOF" | tr -s '[:space:]' - [ - paths(arrays) as $path | { - "key": ($path + ["length"]) | join("_"), - "val": getpath($path) | length - } - ] + [ - paths(scalars) as $path | { - "key": $path | join("_"), - "val": getpath($path) - } - ] - | map(select(.key | startswith("repos"))) - | map(.key + "=" + (.val | @sh | gsub("[\\n]"; " ")))[] -________EOF -)"; + + local query=".repos | {repos: .}"; if test -n "${repo:-}"; then query="$(cat <<________EOF | tr -s '[:space:]' .repos | map(select(.name == "${repo}")) | {repos: .} - | ${query} ________EOF )"; fi - yq -Mo json "${manifest}" | jq -r "${query}"; + rapids-query-manifest ${__rest__[@]} "${query}"; } if test -n "${rapids_build_utils_debug:-}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/query-manifest.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/query-manifest.sh new file mode 100755 index 000000000..e3cb0aabb --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/query-manifest.sh @@ -0,0 +1,49 @@ +#! /usr/bin/env bash + +query_manifest() { + set -euo pipefail; + + local manifest=; + + eval "$( \ + devcontainer-utils-parse-args --names ' + m|manifest | + ' - <<< "${@@Q}" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + manifest="${m:-${manifest:-"${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}"}}"; + + local query=; + + query="$(cat <<"________EOF" | tr -s '[:space:]' + [ + paths(arrays) as $path | { + "key": ($path + ["length"]) | join("_"), + "val": getpath($path) | length + } + ] + [ + paths(scalars) as $path | { + "key": $path | join("_"), + "val": getpath($path) + } + ] + | map(select(.key)) + | map(.key + "=" + (.val | @sh | gsub("[\\n]"; " ")))[] +________EOF +)"; + + query="$(cat <<________EOF | tr -s '[:space:]' + ${__rest__[@]} + | ${query} +________EOF +)"; + + yq -Mo json "${manifest}" | jq -r "${query}"; +} + +if test -n "${rapids_build_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + +(query_manifest "$@"); From 907dbc2be3aeb1ce4e9d716f2ab75471afb10b03 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 11 Jan 2024 12:29:13 -0800 Subject: [PATCH 010/217] cleanup --- .../bin/tmpl/cpp-build.tmpl.sh | 14 +++----- .../bin/tmpl/cpp-clean.tmpl.sh | 2 +- .../bin/tmpl/cpp-configure.tmpl.sh | 32 +++++++++++-------- .../bin/tmpl/python-build-editable.tmpl.sh | 5 +-- .../bin/tmpl/python-build-wheel.tmpl.sh | 1 + 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh index 545844fa4..2259c6acf 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh @@ -4,7 +4,7 @@ build_${CPP_LIB}_cpp() { set -euo pipefail; - if [[ ! -d ~/${CPP_SRC} ]]; then + if [[ ! -d ~/"${CPP_SRC}" ]]; then exit 1; fi @@ -19,23 +19,19 @@ build_${CPP_LIB}_cpp() { verbose="${v:-${verbose:-}}"; - if test -n "${verbose}"; then verbose="--log-level=VERBOSE"; fi - - configure-${CPP_LIB}-cpp ${verbose} ${__rest__[@]}; + configure-${CPP_LIB}-cpp ${verbose:+-v} ${__rest__[@]}; eval "$( \ rapids-get-num-archs-jobs-and-load ${__rest__[@]} \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; - if test -n "${verbose}"; then verbose="--verbose"; fi - time cmake \ - --build ~/${CPP_SRC}/build/latest \ - ${verbose} \ + --build ~/"${CPP_SRC}/build/latest" \ + ${verbose:+--verbose} \ -j${n_jobs} \ -- \ - -l${n_load}; + -l${n_load} ; } if test -n "${rapids_build_utils_debug:-}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh index 131bae46c..2da0ab849 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh @@ -4,7 +4,7 @@ clean_${CPP_LIB}_cpp() { set -euo pipefail; - rm -rf ~/${CPP_SRC}/build/latest/*; + rm -rf ~/"${CPP_SRC}/build/latest"/*; } if test -n "${rapids_build_utils_debug:-}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh index 5fd1082cf..7ca18fb9a 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh @@ -4,7 +4,7 @@ configure_${CPP_LIB}_cpp() { set -euo pipefail; - if [[ ! -d ~/${CPP_SRC} ]]; then + if [[ ! -d ~/"${CPP_SRC}" ]]; then exit 1; fi @@ -12,24 +12,35 @@ configure_${CPP_LIB}_cpp() { local parallel=""; local max_device_obj_memory_usage=""; + local verbose=""; + eval "$( \ devcontainer-utils-parse-args --names ' a|archs | j|parallel | m|max-device-obj-memory-usage | + v|verbose | ' - <<< "$@" \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; + verbose="${v:-${verbose:-}}"; + archs="${a:-${archs:-}}"; - archs="${a:-${archs:+-a${archs}}}"; - parallel="${j:-${parallel:+-j${parallel}}}"; + parallel="${j:-${parallel:-}}"; max_device_obj_memory_usage="${m:-${max_device_obj_memory_usage:-}}"; - max_device_obj_memory_usage="${max_device_obj_memory_usage:+-m${max_device_obj_memory_usage}}"; + + eval "$( \ + rapids-get-num-archs-jobs-and-load \ + ${archs:+-a ${archs}} \ + ${parallel:+-j ${parallel}} \ + ${max_device_obj_memory_usage:+-m ${max_device_obj_memory_usage}} \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; local build_type="$(rapids-parse-cmake-build-type ${__rest__[@]} | tr '[:upper:]' '[:lower:]')"; - local binary_dir=~/${CPP_SRC}/build/${build_type}; - local source_dir=~/${CPP_SRC}; + local binary_dir=~/"${CPP_SRC}/build/${build_type}"; + local source_dir=~/"${CPP_SRC}"; # Reconfigure if previous configure failed if [[ ! -f ${binary_dir}/build.ninja ]]; then @@ -45,16 +56,9 @@ configure_${CPP_LIB}_cpp() { cmake_args+=(${CMAKE_ARGS:-}); cmake_args+=(${CPP_DEPS}); cmake_args+=(${CPP_ARGS}); + cmake_args+=(${verbose:+--log-level=VERBOSE}); cmake_args+=(${__rest__[@]}); - eval "$( \ - rapids-get-num-archs-jobs-and-load \ - ${archs} \ - ${parallel} \ - ${max_device_obj_memory_usage} \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - time CUDAFLAGS="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}" \ cmake ${cmake_args[@]}; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh index 8f74c15fc..478c639ef 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh @@ -41,13 +41,10 @@ build_${PY_LIB}_python_inplace() { local cmake_args=(${PY_CMAKE_ARGS}); - if test -n "${verbose}"; then - cmake_args+=("--log-level=VERBOSE"); - fi - cmake_args+=(${CMAKE_ARGS:-}); cmake_args+=(${CPP_DEPS}); cmake_args+=(${CPP_ARGS}); + cmake_args+=(${verbose:+--log-level=VERBOSE}); cmake_args+=(${__rest__[@]}); local ninja_args=(); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh index c1f9b8144..8aa35bac0 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh @@ -58,6 +58,7 @@ build_${PY_LIB}_python_wheel() { cmake_args+=(${CMAKE_ARGS:-}); cmake_args+=(${CPP_DEPS}); cmake_args+=(${CPP_ARGS}); + cmake_args+=(${verbose:+--log-level=VERBOSE}); cmake_args+=(${__rest__[@]}); local ninja_args=(); From 8e163090c7521a6b9a7946a718ac7deaa1d6ff92 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 11:27:00 -0800 Subject: [PATCH 011/217] sort and dedupe pip dependencies --- .../opt/rapids-build-utils/bin/make-pip-env.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index c71d244d4..5f9d31125 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -24,7 +24,9 @@ make_pip_env() { local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.local/share/venvs/${env_file_name})"; - rapids-make-pip-dependencies ${__rest__[@]} > "${new_env_path}"; + rapids-make-pip-dependencies ${__rest__[@]} \ + | grep -v -e '^$' | tr -s "[:blank:]" | LC_ALL=C sort -u \ + > "${new_env_path}" if test -f "${new_env_path}"; then From b4469d7f37d744a8cc9b5d5acda8460ff5dfdf57 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 11:40:45 -0800 Subject: [PATCH 012/217] constrain cuda-python and cupy versions to the current CUDA version --- .../rapids-build-utils/bin/make-pip-dependencies.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index dedeef342..168156847 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -87,12 +87,12 @@ make_pip_dependencies() { done # Generate a combined requirements.txt file - cat ${pip_reqs_txts[@]} \ - | grep -v '^#' \ - | grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" \ - | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ - | sed -E "s/cupy-cuda[0-9]+x[ ]+?>=.*/cupy-cuda${cuda_version_major}x>=${cuda_version}/g" \ - | sed -E "s/cuda-python[ ]+?>=.*/cuda-python>=${cuda_version},<$((cuda_version_major+1)).0a0/g" \ + cat ${pip_reqs_txts[@]} \ + | grep -v '^#' \ + | grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" \ + | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ + | sed -E "s/cuda-python.*/cuda-python>=${cuda_version},<$((cuda_version_major+1)).0a0/g" \ + | sed -E "s/cupy-cuda[0-9]+x(==|>=|<=|>|<)?.*/cupy-cuda${cuda_version_major}x\1${cuda_version_major}/g" \ ; rm ${pip_reqs_txts[@]}; From b476d9980e43999ac1dc88505c6d704447583204 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 11:43:12 -0800 Subject: [PATCH 013/217] clean compile_commands.json --- .../opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh index 2da0ab849..8b3799b37 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh @@ -4,7 +4,8 @@ clean_${CPP_LIB}_cpp() { set -euo pipefail; - rm -rf ~/"${CPP_SRC}/build/latest"/*; + rm -rf ~/"${CPP_SRC}/build/latest"/* \ + ~/"${CPP_SRC}"/compile_commands.json; } if test -n "${rapids_build_utils_debug:-}"; then From dfc977f940b9f13dbfade684ffbfba53fe0b7121 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 11:43:36 -0800 Subject: [PATCH 014/217] intercept the --mode param and only pass it to the python build script --- .../rapids-build-utils/bin/tmpl/build.tmpl.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh index c612589de..9cc2fdec4 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh @@ -2,14 +2,27 @@ build_${NAME}() { set -euo pipefail; + + local mode="editable"; + + # pull out the --mode editable|wheel arg (if any) + eval "$( \ + devcontainer-utils-parse-args --names ' + m|mode | + ' - <<< "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + mode="${m:-${mode:-"editable"}}"; + for lib in ${CPP_LIB}; do if type build-${lib}-cpp >/dev/null 2>&1; then - build-${lib}-cpp "$@"; + build-${lib}-cpp ${__rest__[@]}; fi done for lib in ${PY_LIB}; do if type build-${lib}-python >/dev/null 2>&1; then - build-${lib}-python "$@"; + build-${lib}-python -m ${mode} ${__rest__[@]}; fi done } From 3d9bc1aac25e53ead301b33a4e19ed08279b75f2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 11:44:22 -0800 Subject: [PATCH 015/217] support defining custom envvars in manifest.yaml python entries --- .../bin/generate-scripts.sh | 5 ++++- .../bin/tmpl/python-build-editable.tmpl.sh | 21 +++++++++++-------- .../bin/tmpl/python-build-wheel.tmpl.sh | 21 +++++++++++-------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index 580b2e0e4..633fa6811 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -21,6 +21,7 @@ generate_script() { cat - \ | envsubst '$NAME $SRC_PATH + $PY_ENV $PY_SRC $PY_LIB $CPP_LIB @@ -152,7 +153,7 @@ generate_scripts() { repo_name="$(tr "[:upper:]" "[:lower:]" <<< "${!repo_name:-}")"; - repo_names+=($repo_name) + repo_names+=($repo_name); # Generate a clone script for each repo ( @@ -230,6 +231,7 @@ generate_scripts() { local py_dirs=() for ((j=0; j < ${!py_length:-0}; j+=1)); do + local py_env="${repo}_python_${j}_env"; local py_name="${repo}_python_${j}_name"; local py_cmake_args="${repo}_python_${j}_args_cmake"; local pip_wheel_args="${repo}_python_${j}_args_wheel"; @@ -243,6 +245,7 @@ generate_scripts() { ( PY_SRC="${py_path}" \ PY_LIB="${!py_name}" \ + PY_ENV="${!py_env:-}" \ CPP_ARGS="${args[@]}" \ CPP_DEPS="${deps[@]}" \ PY_CMAKE_ARGS="${!py_cmake_args:-}" \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh index 478c639ef..03572e1b2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh @@ -78,15 +78,18 @@ build_${PY_LIB}_python_inplace() { local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; - time \ - CUDAFLAGS="${cudaflags}" \ - CMAKE_GENERATOR="Ninja" \ - CMAKE_ARGS="${cmake_args[@]}" \ - SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ - NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ - SETUPTOOLS_ENABLE_FEATURES="legacy-editable" \ - python -m pip install ${pip_args[@]} \ - ; + time ( + CUDAFLAGS="${cudaflags}" \ + CMAKE_GENERATOR="Ninja" \ + CMAKE_ARGS="${cmake_args[@]}" \ + SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ + NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" \ + ${PY_ENV} \ + python -m pip install ${pip_args[@]} \ + ; + { set +x; } 2>/dev/null; echo -n "${PY_LIB} install time:"; + ) 2>&1; } if test -n "${rapids_build_utils_debug:-}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh index 8aa35bac0..d448d3062 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh @@ -130,15 +130,18 @@ build_${PY_LIB}_python_wheel() { local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; - time \ - CUDAFLAGS="${cudaflags}" \ - CMAKE_GENERATOR="Ninja" \ - PARALLEL_LEVEL="${n_jobs}" \ - CMAKE_ARGS="${cmake_args[@]}" \ - SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ - NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ - python -m pip wheel ${pip_args[@]} \ - ; + time ( + CUDAFLAGS="${cudaflags}" \ + CMAKE_GENERATOR="Ninja" \ + PARALLEL_LEVEL="${n_jobs}" \ + CMAKE_ARGS="${cmake_args[@]}" \ + SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ + NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ + ${PY_ENV} \ + python -m pip wheel ${pip_args[@]} \ + ; + { set +x; } 2>/dev/null; echo -n "${PY_LIB} wheel build time:"; + ) 2>&1; } if test -n "${rapids_build_utils_debug:-}"; then From 3a8386280778315c7727fdab58ca98ac5298c45d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 11:54:59 -0800 Subject: [PATCH 016/217] keep cupy package version, just change the cuda modifier --- .../rapids-build-utils/bin/make-pip-dependencies.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 168156847..619b4ba07 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -87,12 +87,12 @@ make_pip_dependencies() { done # Generate a combined requirements.txt file - cat ${pip_reqs_txts[@]} \ - | grep -v '^#' \ - | grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" \ - | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ - | sed -E "s/cuda-python.*/cuda-python>=${cuda_version},<$((cuda_version_major+1)).0a0/g" \ - | sed -E "s/cupy-cuda[0-9]+x(==|>=|<=|>|<)?.*/cupy-cuda${cuda_version_major}x\1${cuda_version_major}/g" \ + cat ${pip_reqs_txts[@]} \ + | grep -v '^#' \ + | grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" \ + | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ + | sed -E "s/cupy-cuda[0-9]+x/cupy-cuda${cuda_version_major}x/g" \ + | sed -E "s/cuda-python.*/cuda-python>=${cuda_version},<$((cuda_version_major+1)).0a0/g" \ ; rm ${pip_reqs_txts[@]}; From b6db4ecb9d48c67080aa868e9fb45e845280aa6b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 17:36:03 -0800 Subject: [PATCH 017/217] clean more setuptools/distutils build dirs --- .../bin/tmpl/python-build-editable.tmpl.sh | 9 ++-- .../bin/tmpl/python-build-wheel.tmpl.sh | 9 ++-- .../bin/tmpl/python-clean.tmpl.sh | 47 +++++++------------ 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh index 03572e1b2..6c9af01f5 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh @@ -75,17 +75,18 @@ build_${PY_LIB}_python_inplace() { trap "rm -rf ~/'${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; - local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; - local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; - time ( + export ${PY_ENV} PATH="$PATH"; + + local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; + local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; + CUDAFLAGS="${cudaflags}" \ CMAKE_GENERATOR="Ninja" \ CMAKE_ARGS="${cmake_args[@]}" \ SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ SETUPTOOLS_ENABLE_FEATURES="legacy-editable" \ - ${PY_ENV} \ python -m pip install ${pip_args[@]} \ ; { set +x; } 2>/dev/null; echo -n "${PY_LIB} install time:"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh index d448d3062..958c0dd2f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh @@ -127,17 +127,18 @@ build_${PY_LIB}_python_wheel() { trap "rm -rf ~/'${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; - local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; - local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; - time ( + export ${PY_ENV} PATH="$PATH"; + + local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; + local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; + CUDAFLAGS="${cudaflags}" \ CMAKE_GENERATOR="Ninja" \ PARALLEL_LEVEL="${n_jobs}" \ CMAKE_ARGS="${cmake_args[@]}" \ SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ - ${PY_ENV} \ python -m pip wheel ${pip_args[@]} \ ; { set +x; } 2>/dev/null; echo -n "${PY_LIB} wheel build time:"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh index cb6b151cc..d5fbf29c9 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh @@ -4,41 +4,30 @@ clean_${PY_LIB}_cpp() { set -euo pipefail; - local py_lib="$(tr '-' '_' <<< "${PY_LIB}")"; + local py_lib="${PY_LIB}"; - rm -rf ~/"${PY_SRC}"/{_skbuild,${py_lib}.egg-info}; + rm -rf ~/"${PY_SRC}"/_skbuild \ + ~/"${PY_SRC}"/{${py_lib},${py_lib//"-"/"_"}}.egg-info; - local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; - python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; - python_version="${python_version/./}"; - - local dir; - for dir in lib temp dist; do - local slug="${dir}.$(uname -s)-$(uname -m)"; - if test -d ~/"${PY_SRC}"/build/${slug,,}-${python_version/./}; then - rm -rf ~/"${PY_SRC}"/build/${slug,,}-${python_version/./}; - fi - if test -d ~/"${PY_SRC}"/build/${slug,,}-cpython-${python_version/./}; then - rm -rf ~/"${PY_SRC}"/build/${slug,,}-cpython-${python_version/./}; + for lib in ${py_lib} ${py_lib//"-"/"_"}; do + if test -d ~/"${PY_SRC}"/${lib}/; then + find ~/"${PY_SRC}"/${lib}/ -type f \ + -iname "*.cpython-*-$(uname -m)-$(uname -s)-*.so" \ + -delete; fi done - if test -d ~/"${PY_SRC}"/build; then - for d in ~/"${PY_SRC}"/build/cp${python_version}-cp${python_version}*; do - rm -rf $d - done - fi + local py_ver="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; + py_ver="$(grep -o '^[0-9]*.[0-9]*' <<< "${py_ver}")"; - if test -d ~/"${PY_SRC}/${PY_LIB}"/; then - find ~/"${PY_SRC}/${PY_LIB}"/ -type f \ - -iname "*.cpython-*-$(uname -m)-$(uname -s)-*.so" \ - -delete; - fi - - if test -d ~/"${PY_SRC}/${py_lib}"/; then - find ~/"${PY_SRC}/${py_lib}"/ -type f \ - -iname "*.cpython-*-$(uname -m)-$(uname -s)-*.so" \ - -delete; + if test -d ~/"${PY_SRC}"/build; then + local slug="$(uname -s)-$(uname -m)"; + rm -rf \ + `# scikit-buld-core build dirs` \ + ~/"${PY_SRC}"/build/cp{${py_ver},${py_ver/./}}-cp{${py_ver},${py_ver/./}}* \ + `# setuptools/distutils build dirs` \ + ~/"${PY_SRC}"/build/{lib,temp,dist,bdist}.${slug,,} \ + ~/"${PY_SRC}"/build/{lib,temp,dist,bdist}.${slug,,}-{,cpython}{,-}{${py_ver},${py_ver/./}}; fi } From fe4657966fe00de2d998bd634702a314b276d9e9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 22:16:16 -0800 Subject: [PATCH 018/217] ignore grep's nonzero exit code Signed-off-by: ptaylor --- .../opt/rapids-build-utils/bin/make-pip-env.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 5f9d31125..7e2e9424f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -24,8 +24,10 @@ make_pip_env() { local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.local/share/venvs/${env_file_name})"; - rapids-make-pip-dependencies ${__rest__[@]} \ - | grep -v -e '^$' | tr -s "[:blank:]" | LC_ALL=C sort -u \ + rapids-make-pip-dependencies ${__rest__[@]} \ + | grep -v -e '^$' || true \ + | tr -s "[:blank:]" \ + | LC_ALL=C sort -u \ > "${new_env_path}" if test -f "${new_env_path}"; then From 2d6c9068646a76f1b433e6dad22e82c4c5dd8ab0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 22:16:38 -0800 Subject: [PATCH 019/217] add cpp install and cpack scripts Signed-off-by: ptaylor --- .../bin/generate-scripts.sh | 2 +- .../bin/tmpl/cpp-cpack.tmpl.sh | 39 +++++++++++++++++++ .../bin/tmpl/cpp-install.tmpl.sh | 16 ++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index 633fa6811..d91fda223 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -93,7 +93,7 @@ generate_repo_scripts() { generate_cpp_scripts() { local script_name; - for script_name in "configure" "build" "clean"; do ( + for script_name in "configure" "build" "clean" "install" "cpack"; do ( cat ${TMPL}/cpp-${script_name}.tmpl.sh \ | CPP_SRC="${SRC_PATH:-}${CPP_SRC:+/$CPP_SRC}" \ generate_script "${script_name}-${CPP_LIB}-cpp"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh new file mode 100755 index 000000000..0a9adfec6 --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh @@ -0,0 +1,39 @@ +#! /usr/bin/env bash + +install_${CPP_LIB}_cpp() { + set -euo pipefail; + + local verbose=""; + + eval "$( \ + devcontainer-utils-parse-args --names ' + v|verbose | + o|out-dir | + ' - <<< "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + verbose="${v:-${verbose:-}}"; + out_dir="${o:-${out_dir:-}}"; + + time ( + ( + cd ~/"${CPP_SRC}"/build/latest/; + cpack -G TGZ ${verbose:+-V}; + ) || true; + + if test -d "${out_dir}"/; then + find ~/"${CPP_SRC}"/build/latest/ \ + -iname "${CPP_LIB}-*.tar.gz" \ + -exec cp -a "{}" "${out_dir}"/; + fi + + { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} CPack time:"; + ) 2>&1; +} + +if test -n "${rapids_build_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + +install_${CPP_LIB}_cpp "$@"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh new file mode 100755 index 000000000..7bffe7a65 --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh @@ -0,0 +1,16 @@ +#! /usr/bin/env bash + +install_${CPP_LIB}_cpp() { + set -euo pipefail; + + time ( + cmake --install ~/"${CPP_SRC}"/build/latest/ "$@"; + { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} install time:"; + ) 2>&1; +} + +if test -n "${rapids_build_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + +install_${CPP_LIB}_cpp "$@"; From 5bb2e4c29fbf782c1cf41bb7dcd2e5a06700802d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 Jan 2024 22:17:03 -0800 Subject: [PATCH 020/217] add a timing label to build and configure steps Signed-off-by: ptaylor --- .../opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh | 7 ++++++- .../opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh index 2259c6acf..9f2029a06 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh @@ -26,12 +26,17 @@ build_${CPP_LIB}_cpp() { | xargs -r -d'\n' -I% echo -n local %\; \ )"; - time cmake \ + # Build C++ lib + time ( + cmake \ --build ~/"${CPP_SRC}/build/latest" \ ${verbose:+--verbose} \ -j${n_jobs} \ -- \ -l${n_load} ; + { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} build time:"; + ) 2>&1; + } if test -n "${rapids_build_utils_debug:-}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh index 7ca18fb9a..e5d68896f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh @@ -59,8 +59,11 @@ configure_${CPP_LIB}_cpp() { cmake_args+=(${verbose:+--log-level=VERBOSE}); cmake_args+=(${__rest__[@]}); - time CUDAFLAGS="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}" \ - cmake ${cmake_args[@]}; + time ( + CUDAFLAGS="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}" \ + cmake ${cmake_args[@]}; + { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} configure time:"; + ) 2>&1; if [[ ! -L ${source_dir}/compile_commands.json \ || "$(readlink "${source_dir}/compile_commands.json")" \ From e14f787757518353eceb295b0ab99f47b437720e Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Sat, 13 Jan 2024 11:43:21 -0800 Subject: [PATCH 021/217] install system dependencies for ucxx, cudf_kafka, and cuspatial in pip containers (#209) Signed-off-by: ptaylor --- .devcontainer/cuda11.8-pip/devcontainer.json | 2 ++ .devcontainer/cuda12.0-pip/devcontainer.json | 2 ++ .devcontainer/rapids.Dockerfile | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index ff8d4f0fb..94212b758 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -10,10 +10,12 @@ }, "hostRequirements": {"gpu": "optional"}, "features": { + "./features/src/ucx": {"version": "1.14.1"}, "./features/src/cuda": {"installcuDNN": true}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ + "./features/src/ucx", "./features/src/cuda", "./features/src/rapids-build-utils" ], diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json index db0763b9f..186498d18 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -10,10 +10,12 @@ }, "hostRequirements": {"gpu": "optional"}, "features": { + "./features/src/ucx": {"version": "1.14.1"}, "./features/src/cuda": {"installcuDNN": true}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ + "./features/src/ucx", "./features/src/cuda", "./features/src/rapids-build-utils" ], diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 9d35e3f97..2400d6aa9 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -5,6 +5,14 @@ ARG PYTHON_PACKAGE_MANAGER=conda FROM ${BASE} as pip-base +RUN apt update -y \ + && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ + # needed by libcudf_kafka + librdkafka-dev \ + # needed by libcuspatial + sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev \ + && rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*; + ENV DEFAULT_VIRTUAL_ENV=rapids FROM ${BASE} as conda-base From 5b58b2f2c7111b49f0277e790a56f7c32ec6d460 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 13 Jan 2024 11:46:17 -0800 Subject: [PATCH 022/217] run rapids-dependency-file-generator in parallel Signed-off-by: ptaylor --- .../bin/make-conda-dependencies.sh | 29 +++++++--- .../bin/make-pip-dependencies.sh | 53 +++++++++++-------- .../rapids-build-utils/bin/make-pip-env.sh | 2 +- 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index edf05a2d2..90b4105b3 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -1,5 +1,13 @@ #! /usr/bin/env bash +generate_env_yaml() { + ( + (rapids-dependency-file-generator ${@:2} 2>/dev/null || echo "name: none") \ + | (grep -v '^#' || [ "$?" == "1" ]) \ + | tee "${1}" 1>/dev/null; + ) & true +} + make_conda_dependencies() { set -euo pipefail; @@ -52,30 +60,37 @@ make_conda_dependencies() { local keyi; for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - if rapids-dependency-file-generator \ + local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.env.yaml"; + conda_env_yamls+=("${file}"); + generate_env_yaml \ + "${file}" \ --file_key ${repo_keys[$keyi]} \ --output conda \ --config ~/"${!repo_path}/dependencies.yaml" \ --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ - `# --stdout` \ - >> /tmp/${!repo_name}.${!py_name}.${keyi}.env.yaml 2>/dev/null; then - conda_env_yamls+=("/tmp/${!repo_name}.${!py_name}.${keyi}.env.yaml"); - fi + ; done done fi done if test ${#conda_env_yamls[@]} -gt 0; then + + for ((i=0; i < ${#conda_env_yamls[@]}; i+=1)); do + while ! test -f "${conda_env_yamls[$i]}"; do + sleep 0.1; + done + done + local conda_noinstall=($(rapids-python-pkg-names) $(rapids-python-conda-pkg-names)); # Generate a combined conda env yaml file. - conda-merge ${conda_env_yamls[@]} \ + conda-merge "${conda_env_yamls[@]}" \ | grep -v '^name:' \ | grep -v -P '^[ ]*?\- (\.git\@[^(main|master)])(.*?)$' \ | grep -v -P "^[ ]*?\- ($(tr -d '[:blank:]' <<< "${conda_noinstall[@]/%/ |}"))(=.*|>.*|<.*)?$" \ ; - rm ${conda_env_yamls[@]}; + rm -f "${conda_env_yamls[@]}"; fi } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 619b4ba07..64635c628 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -1,8 +1,16 @@ #! /usr/bin/env bash +generate_requirements() { + ( + (rapids-dependency-file-generator ${@:2} 2>/dev/null || echo "") \ + | (grep -v '^#' || [ "$?" == "1" ]) \ + | tee "${1}" 1>/dev/null; + ) & true +} + make_pip_dependencies() { - set -euo pipefail; + set -Eeuo pipefail; local keys=(); @@ -19,12 +27,8 @@ make_pip_dependencies() { fi local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; - cuda_version="$(grep -o '^[0-9]*.[0-9]*.[0-9]' <<< "${cuda_version}")"; + cuda_version="$(grep -o '^[0-9]*.[0-9]*' <<< "${cuda_version}")"; local cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")"; - local cuda_version_minor="$(cut -d'.' -f2 <<< "${cuda_version}")"; - local cuda_version_patch="$(cut -d'.' -f3 <<< "${cuda_version}")"; - local cuda_version_mm="${cuda_version_major}.${cuda_version_minor:-0}"; - cuda_version="${cuda_version_mm}.${cuda_version_patch:-0}"; local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; @@ -57,15 +61,15 @@ make_pip_dependencies() { local keyi; for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do - if rapids-dependency-file-generator \ - --file_key ${repo_keys[$keyi]} \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version_mm};py=${python_version}" \ - `# --stdout` \ - >> /tmp/${!repo_name}.${!py_name}.${keyi}.requirements.txt 2>/dev/null; then - pip_reqs_txts+=("/tmp/${!repo_name}.${!py_name}.${keyi}.requirements.txt"); - fi + local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.requirements.txt"; + pip_reqs_txts+=("${file}"); + generate_requirements \ + "${file}" \ + --file_key ${repo_keys[$keyi]} \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + ; done done fi @@ -73,6 +77,12 @@ make_pip_dependencies() { if test ${#pip_reqs_txts[@]} -gt 0; then + for ((i=0; i < ${#pip_reqs_txts[@]}; i+=1)); do + while ! test -f "${pip_reqs_txts[$i]}"; do + sleep 0.1; + done + done + local pip_noinstall=(); # add all python packages to the noinstall list. @@ -87,15 +97,14 @@ make_pip_dependencies() { done # Generate a combined requirements.txt file - cat ${pip_reqs_txts[@]} \ - | grep -v '^#' \ - | grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" \ - | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ - | sed -E "s/cupy-cuda[0-9]+x/cupy-cuda${cuda_version_major}x/g" \ - | sed -E "s/cuda-python.*/cuda-python>=${cuda_version},<$((cuda_version_major+1)).0a0/g" \ + cat "${pip_reqs_txts[@]}" \ + | (grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" || [ "$?" == "1" ]) \ + | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ + | sed -E "s/cupy-cuda[0-9]+x/cupy-cuda${cuda_version_major}x/g" \ + | sed -E "s/cuda-python.*/cuda-python>=${cuda_version}.0,<$((cuda_version_major+1)).0a0/g" \ ; - rm ${pip_reqs_txts[@]}; + rm -f "${pip_reqs_txts[@]}"; fi } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 7e2e9424f..397a90815 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -25,7 +25,7 @@ make_pip_env() { local old_env_path="$(realpath -m ~/.local/share/venvs/${env_file_name})"; rapids-make-pip-dependencies ${__rest__[@]} \ - | grep -v -e '^$' || true \ + | (grep -v -E '^$' || [ "$?" == "1" ]) \ | tr -s "[:blank:]" \ | LC_ALL=C sort -u \ > "${new_env_path}" From dee1b73ee30dad2451e76f4bd76b57e2d1d959d1 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 13 Jan 2024 11:46:54 -0800 Subject: [PATCH 023/217] use absolute paths Signed-off-by: ptaylor --- .../rapids-build-utils/bin/generate-scripts.sh | 18 +++++++++--------- .../rapids-build-utils/bin/tmpl/clone.tmpl.sh | 12 ++++++------ .../bin/tmpl/cpp-build.tmpl.sh | 7 +++---- .../bin/tmpl/cpp-clean.tmpl.sh | 4 ++-- .../bin/tmpl/cpp-configure.tmpl.sh | 8 ++++---- .../bin/tmpl/cpp-cpack.tmpl.sh | 7 ++++--- .../bin/tmpl/cpp-install.tmpl.sh | 5 +++-- .../bin/tmpl/python-build-editable.tmpl.sh | 8 ++++---- .../bin/tmpl/python-build-wheel.tmpl.sh | 8 ++++---- .../bin/tmpl/python-clean.tmpl.sh | 16 ++++++++-------- 10 files changed, 47 insertions(+), 46 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index d91fda223..672ca498a 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -158,7 +158,7 @@ generate_scripts() { # Generate a clone script for each repo ( NAME="${repo_name:-}" \ - SRC_PATH="${!repo_path:-}" \ + SRC_PATH=~/"${!repo_path:-}" \ GIT_TAG="${!git_tag:-}" \ GIT_REPO="${!git_repo:-}" \ GIT_HOST="${!git_host:-}" \ @@ -177,7 +177,7 @@ generate_scripts() { local cpp_args="${repo}_cpp_${j}_args"; local cpp_sub_dir="${repo}_cpp_${j}_sub_dir"; local cpp_depends_length="${repo}_cpp_${j}_depends_length"; - local cpp_path="${!repo_path:-}${!cpp_sub_dir:+/${!cpp_sub_dir}}"; + local cpp_path=~/"${!repo_path:-}${!cpp_sub_dir:+/${!cpp_sub_dir}}"; cpp_dirs+=("${cpp_path}"); cpp_libs+=("${!cpp_name:-}"); @@ -201,12 +201,12 @@ generate_scripts() { done ( - SRC_PATH="${!repo_path:-}" \ - CPP_LIB="${cpp_name:-}" \ - CPP_SRC="${!cpp_sub_dir:-}" \ - CPP_ARGS="${!cpp_args:-}" \ - CPP_DEPS="${deps[@]}" \ - generate_cpp_scripts ; + SRC_PATH=~/"${!repo_path:-}" \ + CPP_LIB="${cpp_name:-}" \ + CPP_SRC="${!cpp_sub_dir:-}" \ + CPP_ARGS="${!cpp_args:-}" \ + CPP_DEPS="${deps[@]}" \ + generate_cpp_scripts ; ) || true; done @@ -238,7 +238,7 @@ generate_scripts() { local pip_install_args="${repo}_python_${j}_args_install"; local py_sub_dir="${repo}_python_${j}_sub_dir"; local py_depends_length="${repo}_python_${j}_depends_length"; - local py_path="${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}"; + local py_path=~/"${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}"; py_libs+=(${!py_name}); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh index c80e320e4..8e569a9be 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh @@ -3,7 +3,7 @@ clone_${NAME}() { set -euo pipefail; - if [[ ! -d ~/"${SRC_PATH}"/.git ]]; then + if [[ ! -d "${SRC_PATH}"/.git ]]; then local branch=; local upstream=; @@ -25,7 +25,7 @@ clone_${NAME}() { )"; branch="${b:-${branch:-"${GIT_TAG}"}}"; - directory="${d:-${directory:-"${HOME}/${SRC_PATH}"}}"; + directory="${d:-${directory:-"${SRC_PATH}"}}"; upstream="${u:-${upstream:-"${GIT_UPSTREAM}/${GIT_REPO}"}}"; echo 'Cloning ${NAME}' 1>&2; @@ -43,14 +43,14 @@ clone_${NAME}() { "${directory}" \ ; - git -C ~/"${SRC_PATH}" config --add remote.upstream.fetch '^refs/heads/pull-request/*'; + git -C "${SRC_PATH}" config --add remote.upstream.fetch '^refs/heads/pull-request/*'; - local upstream_branches="$(git -C ~/"${SRC_PATH}" branch --remotes --list 'upstream/pull-request/*')"; + local upstream_branches="$(git -C "${SRC_PATH}" branch --remotes --list 'upstream/pull-request/*')"; if test -n "${upstream_branches:-}"; then - git -C ~/"${SRC_PATH}" branch --remotes -d ${upstream_branches}; + git -C "${SRC_PATH}" branch --remotes -d ${upstream_branches}; fi - git -C ~/"${SRC_PATH}" remote prune upstream; + git -C "${SRC_PATH}" remote prune upstream; if test -z "${no_update_env:-}"; then rapids-update-content-command; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh index 9f2029a06..4d0def9be 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh @@ -2,9 +2,9 @@ build_${CPP_LIB}_cpp() { - set -euo pipefail; + set -Eeuo pipefail; - if [[ ! -d ~/"${CPP_SRC}" ]]; then + if [[ ! -d "${CPP_SRC}" ]]; then exit 1; fi @@ -29,14 +29,13 @@ build_${CPP_LIB}_cpp() { # Build C++ lib time ( cmake \ - --build ~/"${CPP_SRC}/build/latest" \ + --build "${CPP_SRC}/build/latest" \ ${verbose:+--verbose} \ -j${n_jobs} \ -- \ -l${n_load} ; { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} build time:"; ) 2>&1; - } if test -n "${rapids_build_utils_debug:-}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh index 8b3799b37..f25c83766 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh @@ -4,8 +4,8 @@ clean_${CPP_LIB}_cpp() { set -euo pipefail; - rm -rf ~/"${CPP_SRC}/build/latest"/* \ - ~/"${CPP_SRC}"/compile_commands.json; + rm -rf "${CPP_SRC}/build/latest"/* \ + "${CPP_SRC}"/compile_commands.json; } if test -n "${rapids_build_utils_debug:-}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh index e5d68896f..9b53e5e1f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh @@ -2,9 +2,9 @@ configure_${CPP_LIB}_cpp() { - set -euo pipefail; + set -Eeuo pipefail; - if [[ ! -d ~/"${CPP_SRC}" ]]; then + if [[ ! -d "${CPP_SRC}" ]]; then exit 1; fi @@ -39,8 +39,8 @@ configure_${CPP_LIB}_cpp() { )"; local build_type="$(rapids-parse-cmake-build-type ${__rest__[@]} | tr '[:upper:]' '[:lower:]')"; - local binary_dir=~/"${CPP_SRC}/build/${build_type}"; - local source_dir=~/"${CPP_SRC}"; + local binary_dir="${CPP_SRC}/build/${build_type}"; + local source_dir="${CPP_SRC}"; # Reconfigure if previous configure failed if [[ ! -f ${binary_dir}/build.ninja ]]; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh index 0a9adfec6..be1645ea7 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh @@ -1,7 +1,8 @@ #! /usr/bin/env bash install_${CPP_LIB}_cpp() { - set -euo pipefail; + + set -Eeuo pipefail; local verbose=""; @@ -18,12 +19,12 @@ install_${CPP_LIB}_cpp() { time ( ( - cd ~/"${CPP_SRC}"/build/latest/; + cd "${CPP_SRC}"/build/latest/; cpack -G TGZ ${verbose:+-V}; ) || true; if test -d "${out_dir}"/; then - find ~/"${CPP_SRC}"/build/latest/ \ + find "${CPP_SRC}"/build/latest/ \ -iname "${CPP_LIB}-*.tar.gz" \ -exec cp -a "{}" "${out_dir}"/; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh index 7bffe7a65..bee264bb4 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh @@ -1,10 +1,11 @@ #! /usr/bin/env bash install_${CPP_LIB}_cpp() { - set -euo pipefail; + + set -Eeuo pipefail; time ( - cmake --install ~/"${CPP_SRC}"/build/latest/ "$@"; + cmake --install "${CPP_SRC}"/build/latest/ "$@"; { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} install time:"; ) 2>&1; } diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh index 6c9af01f5..228aca4b2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh @@ -2,9 +2,9 @@ build_${PY_LIB}_python_inplace() { - set -euo pipefail; + set -Eeuo pipefail; - if [[ ! -d ~/"${PY_SRC}" ]]; then + if [[ ! -d "${PY_SRC}" ]]; then exit 1; fi @@ -69,11 +69,11 @@ build_${PY_LIB}_python_inplace() { pip_args+=("--no-build-isolation"); pip_args+=("--no-deps"); pip_args+=("--editable"); - pip_args+=(~/"${PY_SRC}"); + pip_args+=("${PY_SRC}"); cmake_args+=("-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON"); - trap "rm -rf ~/'${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; + trap "rm -rf '${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; time ( export ${PY_ENV} PATH="$PATH"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh index 958c0dd2f..4e7a15470 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh @@ -2,9 +2,9 @@ build_${PY_LIB}_python_wheel() { - set -euo pipefail; + set -Eeuo pipefail; - if [[ ! -d ~/"${PY_SRC}" ]]; then + if [[ ! -d "${PY_SRC}" ]]; then exit 1; fi @@ -123,9 +123,9 @@ build_${PY_LIB}_python_wheel() { pip_args+=("--no-build-isolation"); fi - pip_args+=(~/"${PY_SRC}"); + pip_args+=("${PY_SRC}"); - trap "rm -rf ~/'${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; + trap "rm -rf '${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; time ( export ${PY_ENV} PATH="$PATH"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh index d5fbf29c9..22d7f0bee 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh @@ -6,12 +6,12 @@ clean_${PY_LIB}_cpp() { local py_lib="${PY_LIB}"; - rm -rf ~/"${PY_SRC}"/_skbuild \ - ~/"${PY_SRC}"/{${py_lib},${py_lib//"-"/"_"}}.egg-info; + rm -rf "${PY_SRC}"/_skbuild \ + "${PY_SRC}"/{${py_lib},${py_lib//"-"/"_"}}.egg-info; for lib in ${py_lib} ${py_lib//"-"/"_"}; do - if test -d ~/"${PY_SRC}"/${lib}/; then - find ~/"${PY_SRC}"/${lib}/ -type f \ + if test -d "${PY_SRC}"/${lib}/; then + find "${PY_SRC}"/${lib}/ -type f \ -iname "*.cpython-*-$(uname -m)-$(uname -s)-*.so" \ -delete; fi @@ -20,14 +20,14 @@ clean_${PY_LIB}_cpp() { local py_ver="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; py_ver="$(grep -o '^[0-9]*.[0-9]*' <<< "${py_ver}")"; - if test -d ~/"${PY_SRC}"/build; then + if test -d "${PY_SRC}"/build; then local slug="$(uname -s)-$(uname -m)"; rm -rf \ `# scikit-buld-core build dirs` \ - ~/"${PY_SRC}"/build/cp{${py_ver},${py_ver/./}}-cp{${py_ver},${py_ver/./}}* \ + "${PY_SRC}"/build/cp{${py_ver},${py_ver/./}}-cp{${py_ver},${py_ver/./}}* \ `# setuptools/distutils build dirs` \ - ~/"${PY_SRC}"/build/{lib,temp,dist,bdist}.${slug,,} \ - ~/"${PY_SRC}"/build/{lib,temp,dist,bdist}.${slug,,}-{,cpython}{,-}{${py_ver},${py_ver/./}}; + "${PY_SRC}"/build/{lib,temp,dist,bdist}.${slug,,} \ + "${PY_SRC}"/build/{lib,temp,dist,bdist}.${slug,,}-{,cpython}{,-}{${py_ver},${py_ver/./}}; fi } From 74f027f44b131373cf61950471ad335240b8fb4a Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 16 Jan 2024 19:47:37 -0800 Subject: [PATCH 024/217] fix find command --- .../opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh index be1645ea7..ee6909acb 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash -install_${CPP_LIB}_cpp() { +cpack_${CPP_LIB}_cpp() { set -Eeuo pipefail; @@ -24,9 +24,9 @@ install_${CPP_LIB}_cpp() { ) || true; if test -d "${out_dir}"/; then - find "${CPP_SRC}"/build/latest/ \ - -iname "${CPP_LIB}-*.tar.gz" \ - -exec cp -a "{}" "${out_dir}"/; + find "${CPP_SRC}"/build/latest/ \ + -iname "${CPP_LIB}-*.tar.gz" \ + -exec cp -a "{}" "${out_dir}"/ \; fi { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} CPack time:"; @@ -37,4 +37,4 @@ if test -n "${rapids_build_utils_debug:-}"; then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi -install_${CPP_LIB}_cpp "$@"; +cpack_${CPP_LIB}_cpp "$@"; From b568c49ef052a39b5464d97250dc274d710c283b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 16 Jan 2024 19:47:49 -0800 Subject: [PATCH 025/217] allow passing extra requirements --- .../opt/rapids-build-utils/bin/make-pip-dependencies.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 64635c628..bdd775112 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -13,10 +13,12 @@ make_pip_dependencies() { set -Eeuo pipefail; local keys=(); + local requirements=(); eval "$( \ devcontainer-utils-parse-args --names ' k|keys | + r|requirement | ' - <<< "$@" \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; @@ -26,6 +28,9 @@ make_pip_dependencies() { keys=(py_build py_run py_test all); fi + if test -v r; then requirements+=(${r[@]}); fi + if test -v requirement; then requirements+=(${requirement[@]}); fi + local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; cuda_version="$(grep -o '^[0-9]*.[0-9]*' <<< "${cuda_version}")"; local cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")"; @@ -97,7 +102,7 @@ make_pip_dependencies() { done # Generate a combined requirements.txt file - cat "${pip_reqs_txts[@]}" \ + cat "${requirements[@]}" "${pip_reqs_txts[@]}" \ | (grep -v -P "^($(tr -d '[:blank:]' <<< "${pip_noinstall[@]/%/|}"))(=.*|>.*|<.*)?$" || [ "$?" == "1" ]) \ | sed -E "s/-cu([0-9]+)/-cu${cuda_version_major}/g" \ | sed -E "s/cupy-cuda[0-9]+x/cupy-cuda${cuda_version_major}x/g" \ From a8b1313b507df4640776b42d3faf901a1dcd6224 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 12:56:05 -0800 Subject: [PATCH 026/217] add devcontainer-utils-parse-args-from-docstring script --- .../bin/parse-args-from-docstring.sh | 48 +++++++++++++++++++ .../utils/opt/devcontainer/bin/parse-args.sh | 10 ++-- 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100755 features/src/utils/opt/devcontainer/bin/parse-args-from-docstring.sh diff --git a/features/src/utils/opt/devcontainer/bin/parse-args-from-docstring.sh b/features/src/utils/opt/devcontainer/bin/parse-args-from-docstring.sh new file mode 100755 index 000000000..5651568ce --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/parse-args-from-docstring.sh @@ -0,0 +1,48 @@ +#! /usr/bin/env bash + +if test -n "${devcontainer_utils_debug:-}" \ +&& ( test -z "${devcontainer_utils_debug##*"all"*}" \ + || test -z "${devcontainer_utils_debug##*"parse-args"*}" ); then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + +print_usage() { + sed -n '2,/^$/p' "${BASH_SOURCE[${#BASH_SOURCE[@]}-1]}" | sed -r 's/^# ?//'; +} + +parse_names_from_usage() { + cat - \ + | sed -rn 's/^([ ]*)(--?[^ ]+|,|_|\*)+(.*)$/\2/p' \ + | sed -r 's/--?([^ ,]+)+,?/\1|/g' \ + | sed -r 's/\*/\\*/g'; +} + +parse_bool_names_from_usage() { + cat - \ + | sed -rn 's/^([ ]*)(--?[^ ]+|,|_)+([^<]*)$/\2/p' \ + | sed -r 's/(,|\|)/ /g'; +} + +parse_value_names_from_usage() { + cat - \ + | sed -rn 's/^([ ]*)(--?[^ \*]+|,|_)+[ ]*<(.[^>]*).*$/\2/p' \ + | sed -r 's/(,|\|)/ /g'; +} + +parse_value_types_from_usage() { + cat - \ + | sed -rn 's/^([ ]*)(--?[^ \*]+|,|_)+[ ]*<(.[^>]*).*$/\3/p'; +} + +parse_args_or_show_help() { + eval "$( \ + devcontainer-utils-parse-args \ + --names "$(print_usage | parse_names_from_usage)" \ + "$@" <&0 \ + | xargs -r -d'\n' -I% echo -n export %\; \ + )"; + if test -n "${h:-${help:-${usage:-}}}"; then + print_usage >&2; + exit 0; + fi +} diff --git a/features/src/utils/opt/devcontainer/bin/parse-args.sh b/features/src/utils/opt/devcontainer/bin/parse-args.sh index f04106a48..74f2732b1 100755 --- a/features/src/utils/opt/devcontainer/bin/parse-args.sh +++ b/features/src/utils/opt/devcontainer/bin/parse-args.sh @@ -11,8 +11,8 @@ parse_args() { local vars_array=(); if [ "${1:-}" = "--names" ]; then shift; - vars="$(tr -d '[:space:]' <<< "${1}")"; - vars_array=($(tr '|' ' ' <<< "${vars}")); + vars="$(tr -d '[:space:]' <<< "${1}" | rev | cut -d'|' -f1 --complement | rev)"; + vars_array=("${vars//|/ }"); readarray -t vars_array < <( for str in "${vars_array[@]}"; do printf '%d\t%s\n' "${#str}" "$str" @@ -78,6 +78,8 @@ parse_args() { else val="true"; fi + else + rest+=("${arg@Q}"); fi else rest+=("${@@Q}"); @@ -121,7 +123,9 @@ parse_args() { done } -if test -n "${devcontainer_utils_debug:-}"; then +if test -n "${devcontainer_utils_debug:-}" \ +&& ( test -z "${devcontainer_utils_debug##*"all"*}" \ + || test -z "${devcontainer_utils_debug##*"parse-args"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi From fb8482c5d786b8017ad9da31d9fdf6d42277b643 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 12:57:45 -0800 Subject: [PATCH 027/217] add bash completion template --- features/src/utils/install.sh | 62 ++++++---- .../devcontainer/bin/bash/completion.tmpl.sh | 108 ++++++++++++++++++ 2 files changed, 149 insertions(+), 21 deletions(-) create mode 100755 features/src/utils/opt/devcontainer/bin/bash/completion.tmpl.sh diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index 6c4496443..2e8d8f6b5 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -62,34 +62,54 @@ find /opt/devcontainer \ -o -type f -exec chmod 0755 {} \; \); install_utility() { - update-alternatives --install "/usr/bin/$1" "$1" "/opt/devcontainer/bin/$2" 0; + local cmd="devcontainer-utils-${1}"; + local src="${2:-"${1}.sh"}"; + # Install alternative + update-alternatives --install /usr/bin/${cmd} ${cmd} /opt/devcontainer/bin/${src} 0; + + # Install bash_completion script + cat "$(which devcontainer-utils-bash-completion.tmpl)" \ + | SCRIPT="${cmd}" \ + NAME="${cmd//-/_}" \ + envsubst '$SCRIPT $NAME' \ + | tee "/etc/bash_completion.d/${cmd}" >/dev/null \ + ; } -install_utility devcontainer-utils-parse-args parse-args.sh; -install_utility devcontainer-utils-shell-is-interactive shell-is-interactive.sh; -install_utility devcontainer-utils-post-attach-command post-attach-command.sh; -install_utility devcontainer-utils-post-attach-command-entrypoint post-attach-command-entrypoint.sh; -install_utility devcontainer-utils-python-repl-startup python-repl-startup.py; -install_utility devcontainer-utils-init-git git/init.sh; -install_utility devcontainer-utils-clone-git-repo git/repo/clone.sh; +# Install alternatives -install_utility devcontainer-utils-init-ssh-deploy-keys ssh/init-deploy-keys.sh; +# Install this first since we use it in `install_utility` +update-alternatives --install \ + /usr/bin/devcontainer-utils-bash-completion.tmpl \ + devcontainer-utils-bash-completion.tmpl \ + /opt/devcontainer/bin/bash/completion.tmpl.sh 0; -install_utility devcontainer-utils-init-github-cli github/cli/init.sh; -install_utility devcontainer-utils-clone-github-repo github/repo/clone.sh; +install_utility parse-args parse-args.sh; +install_utility parse-args-from-docstring parse-args-from-docstring.sh; +install_utility shell-is-interactive shell-is-interactive.sh; +install_utility post-attach-command post-attach-command.sh; +install_utility post-attach-command-entrypoint post-attach-command-entrypoint.sh; +install_utility python-repl-startup python-repl-startup.py; +install_utility init-git git/init.sh; +install_utility clone-git-repo git/repo/clone.sh; -install_utility devcontainer-utils-init-gitlab-cli gitlab/cli/init.sh; -install_utility devcontainer-utils-clone-gitlab-repo gitlab/repo/clone.sh; -install_utility devcontainer-utils-print-missing-gitlab-token-warning gitlab/print-missing-token-warning.sh; +install_utility init-ssh-deploy-keys ssh/init-deploy-keys.sh; -install_utility devcontainer-utils-vault-auth-github vault/auth/github.sh; +install_utility init-github-cli github/cli/init.sh; +install_utility clone-github-repo github/repo/clone.sh; -install_utility devcontainer-utils-vault-s3-init vault/s3/init.sh; -install_utility devcontainer-utils-vault-s3-creds-generate vault/s3/creds/generate.sh; -install_utility devcontainer-utils-vault-s3-creds-persist vault/s3/creds/persist.sh; -install_utility devcontainer-utils-vault-s3-creds-propagate vault/s3/creds/propagate.sh; -install_utility devcontainer-utils-vault-s3-creds-schedule vault/s3/creds/schedule.sh; -install_utility devcontainer-utils-vault-s3-creds-test vault/s3/creds/test.sh; +install_utility init-gitlab-cli gitlab/cli/init.sh; +install_utility clone-gitlab-repo gitlab/repo/clone.sh; +install_utility print-missing-gitlab-token-warning gitlab/print-missing-token-warning.sh; + +install_utility vault-auth-github vault/auth/github.sh; + +install_utility vault-s3-init vault/s3/init.sh; +install_utility vault-s3-creds-generate vault/s3/creds/generate.sh; +install_utility vault-s3-creds-persist vault/s3/creds/persist.sh; +install_utility vault-s3-creds-propagate vault/s3/creds/propagate.sh; +install_utility vault-s3-creds-schedule vault/s3/creds/schedule.sh; +install_utility vault-s3-creds-test vault/s3/creds/test.sh; # Enable GCC colors for_each_user_bashrc 'sed -i -re "s/^#(export GCC_COLORS)/\1/g" "$0"'; diff --git a/features/src/utils/opt/devcontainer/bin/bash/completion.tmpl.sh b/features/src/utils/opt/devcontainer/bin/bash/completion.tmpl.sh new file mode 100755 index 000000000..4b7f060af --- /dev/null +++ b/features/src/utils/opt/devcontainer/bin/bash/completion.tmpl.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +. devcontainer-utils-parse-args-from-docstring; + +_zip_lines() { + local line_1=""; + local args_1=(); + local line_2=""; + while read -r line_1; do + args_1=(${line_1}); + while read -r line_2; do + local idx; + for ((idx=0; idx < ${#args_1[@]}; idx+=1)); do + echo "${line_2}"; + done + break; + done < "${2}"; + done < "${1}"; +} + +${NAME}_script=""; +${NAME}_usage=""; +${NAME}_bool_names=(); +${NAME}_value_names=(); +${NAME}_value_types=""; +${NAME}_option_names=""; + +_${NAME}_completions() { + # trap 'sleep 10;' EXIT; + + COMPREPLY=(); + + if test -z "${${NAME}_script}"; then + ${NAME}_script="$(which ${SCRIPT})"; + if test -z "${${NAME}_script}"; then return; fi + fi + if test -z "${${NAME}_usage}"; then + ${NAME}_usage="$(sed -n '2,/^$/p' "${${NAME}_script}" | sed -r 's/^# ?//')"; + if test -z "${${NAME}_usage}"; then return; fi + fi + if test "${#${NAME}_bool_names[@]}" -eq 0; then + ${NAME}_bool_names=($(parse_bool_names_from_usage <<< "${${NAME}_usage}")); + fi + if test "${#${NAME}_value_names[@]}" -eq 0; then + ${NAME}_value_names=($(parse_value_names_from_usage <<< "${${NAME}_usage}")); + fi + if test -z "${${NAME}_value_types}"; then + ${NAME}_value_types="$(_zip_lines \ + <(parse_value_names_from_usage <<< "${${NAME}_usage}") \ + <(parse_value_types_from_usage <<< "${${NAME}_usage}") \ + )"; + fi + if test -z "${${NAME}_option_names}"; then + ${NAME}_option_names="${${NAME}_bool_names[*]} ${${NAME}_value_names[*]}"; + fi + + local type_; + local idx_a=0; + local idx_b=0; + + if test "${COMP_CWORD}" -gt 1; then + for ((idx_a=0; idx_a < ${#${NAME}_value_names[@]}; idx_a+=1)); do + if [[ ${${NAME}_value_names[$idx_a]} == ${COMP_WORDS[$COMP_CWORD-1]}* ]]; then + while read -r type_; do + if test ${idx_b} != ${idx_a}; then + idx_b=$((idx_b + 1)); + continue; + fi + case "${type_}" in + num) + COMPREPLY=($(grep -P "^[0-9]+$" <<< "${COMP_WORDS[$COMP_CWORD]}")); + ;; + dir) + COMPREPLY=($(compgen -o nosort -o dirnames -- "${COMP_WORDS[$COMP_CWORD]}")); + ;; + file) + COMPREPLY=($(compgen -o nosort -o filenames -- "${COMP_WORDS[$COMP_CWORD]}")); + ;; + path) + COMPREPLY=($(compgen -o nosort -o default -- "${COMP_WORDS[$COMP_CWORD]}")); + ;; + *) + COMPREPLY=(); + ;; + esac + break; + done <<< "${${NAME}_value_types}"; + return; + fi + done + fi + + for ((idx_a=0; idx_a < ${#${NAME}_bool_names[@]}; idx_a+=1)); do + if [[ ${${NAME}_bool_names[$idx_a]} == ${COMP_WORDS[$COMP_CWORD]}* ]]; then + COMPREPLY=($(compgen -o nosort -W "${${NAME}_option_names}" -- "${COMP_WORDS[$COMP_CWORD]}")); + return; + fi + done + + for ((idx_a=0; idx_a < ${#${NAME}_value_names[@]}; idx_a+=1)); do + if [[ ${${NAME}_value_names[$idx_a]} == ${COMP_WORDS[$COMP_CWORD]}* ]]; then + COMPREPLY=($(compgen -o nosort -W "${${NAME}_option_names}" -- "${COMP_WORDS[$COMP_CWORD]}")); + return; + fi + done +} + +complete -F _${NAME}_completions ${SCRIPT}; From a3ef3ae6dabbe83cb1a69d7d20c0c2f25de00e43 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 12:58:11 -0800 Subject: [PATCH 028/217] generate completions for rapids-build-utils scripts --- features/src/rapids-build-utils/install.sh | 61 ++++++++++++++-------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/features/src/rapids-build-utils/install.sh b/features/src/rapids-build-utils/install.sh index 556b092ec..29dc5064d 100644 --- a/features/src/rapids-build-utils/install.sh +++ b/features/src/rapids-build-utils/install.sh @@ -31,29 +31,44 @@ fi cp -ar ./opt/rapids-build-utils /opt/; -update-alternatives --install /usr/bin/rapids-update-content-command rapids-update-content-command /opt/rapids-build-utils/bin/update-content-command.sh 0; -update-alternatives --install /usr/bin/rapids-post-start-command rapids-post-start-command /opt/rapids-build-utils/bin/post-start-command.sh 0; -update-alternatives --install /usr/bin/rapids-post-attach-command rapids-post-attach-command /opt/rapids-build-utils/bin/post-attach-command.sh 0; -update-alternatives --install /usr/bin/rapids-checkout-same-branch rapids-checkout-same-branch /opt/rapids-build-utils/bin/checkout-same-branch.sh 0; -update-alternatives --install /usr/bin/rapids-pull-repositories rapids-pull-repositories /opt/rapids-build-utils/bin/pull-repositories.sh 0; -update-alternatives --install /usr/bin/rapids-push-repositories rapids-push-repositories /opt/rapids-build-utils/bin/push-repositories.sh 0; -update-alternatives --install /usr/bin/rapids-generate-scripts rapids-generate-scripts /opt/rapids-build-utils/bin/generate-scripts.sh 0; -update-alternatives --install /usr/bin/rapids-join-strings rapids-join-strings /opt/rapids-build-utils/bin/join-strings.sh 0; -update-alternatives --install /usr/bin/rapids-make-conda-dependencies rapids-make-conda-dependencies /opt/rapids-build-utils/bin/make-conda-dependencies.sh 0; -update-alternatives --install /usr/bin/rapids-make-conda-env rapids-make-conda-env /opt/rapids-build-utils/bin/make-conda-env.sh 0; -update-alternatives --install /usr/bin/rapids-make-pip-dependencies rapids-make-pip-dependencies /opt/rapids-build-utils/bin/make-pip-dependencies.sh 0; -update-alternatives --install /usr/bin/rapids-make-pip-env rapids-make-pip-env /opt/rapids-build-utils/bin/make-pip-env.sh 0; -update-alternatives --install /usr/bin/rapids-make-vscode-workspace rapids-make-vscode-workspace /opt/rapids-build-utils/bin/make-vscode-workspace.sh 0; -update-alternatives --install /usr/bin/rapids-parse-cmake-args rapids-parse-cmake-args /opt/rapids-build-utils/bin/parse-cmake-args.sh 0; -update-alternatives --install /usr/bin/rapids-parse-cmake-build-type rapids-parse-cmake-build-type /opt/rapids-build-utils/bin/parse-cmake-build-type.sh 0; -update-alternatives --install /usr/bin/rapids-parse-cmake-var-from-args rapids-parse-cmake-var-from-args /opt/rapids-build-utils/bin/parse-cmake-var-from-args.sh 0; -update-alternatives --install /usr/bin/rapids-parse-cmake-vars-from-args rapids-parse-cmake-vars-from-args /opt/rapids-build-utils/bin/parse-cmake-vars-from-args.sh 0; -update-alternatives --install /usr/bin/rapids-python-pkg-roots rapids-python-pkg-roots /opt/rapids-build-utils/bin/python-pkg-roots.sh 0; -update-alternatives --install /usr/bin/rapids-python-pkg-names rapids-python-pkg-names /opt/rapids-build-utils/bin/python-pkg-names.sh 0; -update-alternatives --install /usr/bin/rapids-python-conda-pkg-names rapids-python-conda-pkg-names /opt/rapids-build-utils/bin/python-conda-pkg-names.sh 0; -update-alternatives --install /usr/bin/rapids-get-num-archs-jobs-and-load rapids-get-num-archs-jobs-and-load /opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh 0; -update-alternatives --install /usr/bin/rapids-list-repos rapids-list-repos /opt/rapids-build-utils/bin/list-repos.sh 0; -update-alternatives --install /usr/bin/rapids-query-manifest rapids-query-manifest /opt/rapids-build-utils/bin/query-manifest.sh 0; +install_utility() { + local cmd="rapids-${1}"; + local src="${2:-"${1}.sh"}"; + # Install alternative + update-alternatives --install /usr/bin/${cmd} ${cmd} /opt/rapids-build-utils/bin/${src} 0; + + # Install bash_completion script + cat "$(which devcontainer-utils-bash-completion.tmpl)" \ + | SCRIPT="${cmd}" \ + NAME="${cmd//-/_}" \ + envsubst '$SCRIPT $NAME' \ + | tee "/etc/bash_completion.d/${cmd}" >/dev/null \ + ; +} + +install_utility update-content-command; +install_utility post-start-command; +install_utility post-attach-command; + +install_utility checkout-same-branch; +install_utility pull-repositories; +install_utility push-repositories; +install_utility generate-scripts; +install_utility make-conda-dependencies; +install_utility make-conda-env; +install_utility make-pip-dependencies; +install_utility make-pip-env; +install_utility make-vscode-workspace; +install_utility parse-cmake-args; +install_utility parse-cmake-build-type; +install_utility parse-cmake-var-from-args; +install_utility parse-cmake-vars-from-args; +install_utility python-pkg-roots; +install_utility python-pkg-names; +install_utility python-conda-pkg-names; +install_utility get-num-archs-jobs-and-load; +install_utility list-repos; +install_utility query-manifest; find /opt/rapids-build-utils \ \( -type d -exec chmod 0775 {} \; \ From bb5909efd8aba0e995147c1159a8c9e6ef0bc4e8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 13:00:37 -0800 Subject: [PATCH 029/217] add docstrings for clone-{git,github,gitlab}-repo scripts --- .../opt/devcontainer/bin/git/repo/clone.sh | 51 +++++++++++----- .../opt/devcontainer/bin/github/repo/clone.sh | 60 +++++++++++++------ .../opt/devcontainer/bin/gitlab/repo/clone.sh | 60 +++++++++++++------ 3 files changed, 117 insertions(+), 54 deletions(-) diff --git a/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh index eb871bff3..7405f4225 100755 --- a/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh @@ -1,22 +1,38 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-clone-git-repo [OPTION]... +# +# Clone a GitLab repository for the logged in user (as reported by `glab auth status`). +# +# If the user doesn't have a fork of the repository, notify the user and ask whether they would like to fork it. +# +# Boolean options: +# -h,--help,--usage print this text +# --no-fork don't prompt the user to fork the repo if a user fork isn't found +# (default: false) +# --no-update-env don't update the Python env with the repo's dependencies after cloning +# (default: false) +# --clone-upstream always clone the upstream, not the user's fork +# (default: false) +# +# Options that require values: +# -b,--branch check the repo out to +# (default: `${NAME}.git.tag` in manifest.yaml) +# -u,--upstream set as the `upstream` remote +# +# Positional arguments: +# directory clone the repo into + +. devcontainer-utils-parse-args-from-docstring; clone_git_repo() { + set -Eeuo pipefail; - set -euo pipefail; + parse_args_or_show_help - <<< "$@"; - local branch=""; - local upstream=""; - - eval "$( \ - devcontainer-utils-parse-args --names ' - b|branch | - u|upstream | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - branch="${b:-"${branch:-}"}"; - upstream="${u:-"${upstream:-}"}"; + local branch="${b:-"${branch:-}"}"; + local upstream="${u:-"${upstream:-}"}"; local nargs="${#__rest__[@]}"; local origin="${__rest__[$((nargs - 2))]}"; @@ -79,7 +95,10 @@ clone_git_repo() { git -C "${directory}" submodule update --init --recursive; } -if test -n "${devcontainer_utils_debug:-}"; then +if test -n "${devcontainer_utils_debug:-}" \ +&& ( test -z "${devcontainer_utils_debug##*"all"*}" \ + || test -z "${devcontainer_utils_debug##*"clone"*}" \ + || test -z "${devcontainer_utils_debug##*"clone-git-repo"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh index fc2f1e870..c5eb28088 100755 --- a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh @@ -1,4 +1,29 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-clone-github-repo [OPTION]... +# +# Clone a GitHub repository for the logged in user (as reported by `gh auth status`). +# +# If the user doesn't have a fork of the repository, notify the user and ask whether they would like to fork it. +# +# Boolean options: +# -h,--help,--usage print this text +# --no-fork don't prompt the user to fork the repo if a user fork isn't found +# (default: false) +# --no-update-env don't update the Python env with the repo's dependencies after cloning +# (default: false) +# --clone-upstream always clone the upstream, not the user's fork +# (default: false) +# +# Options that require values: +# -b,--branch check the repo out to +# +# Positional arguments: +# upstream set as the `upstream` remote +# directory clone the repo into + +. devcontainer-utils-parse-args-from-docstring; get_default_branch() { local repo="${1}"; @@ -56,29 +81,17 @@ ________EOF } clone_github_repo() { + set -Eeuo pipefail; - set -euo pipefail; - - source devcontainer-utils-init-github-cli; - - local branch=; - local no_fork=; - local clone_upstream=; - - eval "$( \ - devcontainer-utils-parse-args --names ' - b|branch | - no-fork | - clone-upstream | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; + parse_args_or_show_help - <<< "$@"; local nargs="${#__rest__[@]}"; local upstream="${__rest__[$((nargs - 2))]}"; upstream="${upstream:?upstream is required}"; - branch="${b:-"${branch:-"$(get_default_branch "${upstream}")"}"}"; + local no_fork="${no_fork:-}"; + local clone_upstream="${clone_upstream:-}"; + local branch="${b:-"${branch:-"$(get_default_branch "${upstream}")"}"}"; local directory="${__rest__[$((nargs - 1))]}"; directory="${directory:?directory is required}"; @@ -92,6 +105,12 @@ clone_github_repo() { local fork=; local owner=; + if test -z "${clone_upstream:-}" || \ + (test -z "${no_fork:-}" && devcontainer-utils-shell-is-interactive); then + source devcontainer-utils-init-github-cli; + user="${GITHUB_USER:-}"; + fi + if test -z "${clone_upstream:-}"; then name="$(get_repo_name "${upstream}")"; owner="$(get_repo_owner "${upstream}")"; @@ -133,7 +152,10 @@ clone_github_repo() { "${origin}" "${directory}" ; } -if test -n "${devcontainer_utils_debug:-}"; then +if test -n "${devcontainer_utils_debug:-}" \ +&& ( test -z "${devcontainer_utils_debug##*"all"*}" \ + || test -z "${devcontainer_utils_debug##*"clone"*}" \ + || test -z "${devcontainer_utils_debug##*"clone-github-repo"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh index 55d1b161c..051cfd2a4 100755 --- a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh @@ -1,4 +1,29 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# devcontainer-utils-clone-gitlab-repo [OPTION]... +# +# Clone a GitLab repository for the logged in user (as reported by `glab auth status`). +# +# If the user doesn't have a fork of the repository, notify the user and ask whether they would like to fork it. +# +# Boolean options: +# -h,--help,--usage print this text +# --no-fork don't prompt the user to fork the repo if a user fork isn't found +# (default: false) +# --no-update-env don't update the Python env with the repo's dependencies after cloning +# (default: false) +# --clone-upstream always clone the upstream, not the user's fork +# (default: false) +# +# Options that require values: +# -b,--branch check the repo out to +# +# Positional arguments: +# upstream set as the `upstream` remote +# directory clone the repo into + +. devcontainer-utils-parse-args-from-docstring; get_default_branch() { local repo="${1}"; @@ -82,29 +107,17 @@ get_user_fork_name() { } clone_gitlab_repo() { + set -Eeuo pipefail; - set -euo pipefail; - - source devcontainer-utils-init-gitlab-cli; - - local branch=; - local no_fork=; - local clone_upstream=; - - eval "$( \ - devcontainer-utils-parse-args --names ' - b|branch | - no-fork | - clone-upstream | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; + parse_args_or_show_help - <<< "$@"; local nargs="${#__rest__[@]}"; local upstream="${__rest__[$((nargs - 2))]}"; upstream="${upstream:?upstream is required}"; - branch="${b:-"${branch:-"$(get_default_branch "${upstream}")"}"}"; + local no_fork="${no_fork:-}"; + local clone_upstream="${clone_upstream:-}"; + local branch="${b:-"${branch:-"$(get_default_branch "${upstream}")"}"}"; local directory="${__rest__[$((nargs - 1))]}"; directory="${directory:?directory is required}"; @@ -118,6 +131,12 @@ clone_gitlab_repo() { local user=; local owner=; + if test -z "${clone_upstream:-}" || \ + (test -z "${no_fork:-}" && devcontainer-utils-shell-is-interactive); then + source devcontainer-utils-init-gitlab-cli; + user="${GITLAB_USER:-}"; + fi + if test -z "${clone_upstream:-}"; then name="$(get_repo_name "${upstream}")"; owner="$(get_repo_owner "${upstream}")"; @@ -159,7 +178,10 @@ clone_gitlab_repo() { "${origin}" "${directory}" ; } -if test -n "${devcontainer_utils_debug:-}"; then +if test -n "${devcontainer_utils_debug:-}" \ +&& ( test -z "${devcontainer_utils_debug##*"all"*}" \ + || test -z "${devcontainer_utils_debug##*"clone"*}" \ + || test -z "${devcontainer_utils_debug##*"clone-gitlab-repo"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi From 6de037c445ac07ec38d2d5c436085eb5718e8dfc Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 13:03:46 -0800 Subject: [PATCH 030/217] format shebangs --- .../opt/rapids-build-utils/bin/join-strings.sh | 12 ------------ .../opt/rapids-build-utils/bin/parse-cmake-args.sh | 2 +- .../bin/parse-cmake-build-type.sh | 2 +- .../bin/parse-cmake-var-from-args.sh | 2 +- .../bin/parse-cmake-vars-from-args.sh | 13 +++++++++++-- .../opt/rapids-build-utils/bin/parse-valid-args.sh | 2 +- .../rapids-build-utils/bin/post-attach-command.sh | 2 +- .../rapids-build-utils/bin/post-start-command.sh | 2 +- .../bin/update-content-command.sh | 2 +- 9 files changed, 18 insertions(+), 21 deletions(-) delete mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/join-strings.sh diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/join-strings.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/join-strings.sh deleted file mode 100755 index 5eaf82466..000000000 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/join-strings.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /usr/bin/env bash - -join_with_delimiter() { - set -euo pipefail - local IFS='' delim=${1:-}; - shift; - echo -n "${1:-}"; - shift; - echo -n "${*/#/$delim}"; -} - -(join_with_delimiter "$@"); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-args.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-args.sh index a92c68872..d3d908d11 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-args.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-args.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash parse_cmake_args() { diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-build-type.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-build-type.sh index 75f1e71ab..3c748f946 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-build-type.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-build-type.sh @@ -1,3 +1,3 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash (rapids-parse-cmake-var-from-args CMAKE_BUILD_TYPE "$@"); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-var-from-args.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-var-from-args.sh index f818a36de..6433c99d8 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-var-from-args.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-var-from-args.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash parse_cmake_var_from_args() { set -euo pipefail; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-vars-from-args.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-vars-from-args.sh index af6346fcb..d78663ad7 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-vars-from-args.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-cmake-vars-from-args.sh @@ -1,11 +1,20 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +join_with_delimiter() { + set -euo pipefail + local IFS='' delim=${1:-}; + shift; + echo -n "${1:-}"; + shift; + echo -n "${*/#/$delim}"; +} parse_cmake_vars_from_args() { set -euo pipefail; local args=""; args="$(rapids-parse-cmake-args "$@")"; - args="$(rapids-join-strings "\n" $args)"; + args="$(join_with_delimiter "\n" $args)"; echo -e "$args" \ | grep '\-D' \ | sed -r 's/^-D//'; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-valid-args.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-valid-args.sh index fdf5af287..0a38d9781 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-valid-args.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/parse-valid-args.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash parse_valid_args() { diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-attach-command.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-attach-command.sh index a13de1005..d4366187e 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-attach-command.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-attach-command.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash if test -z "${SKIP_RAPIDS_BUILD_UTILS_POST_ATTACH_COMMAND:-}"; then if test -n "${PYTHON_PACKAGE_MANAGER:-}"; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh index 2e2d487fd..c1807f408 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/post-start-command.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash if test -z "${SKIP_RAPIDS_BUILD_UTILS_POST_START_COMMAND:-}"; then rapids-make-vscode-workspace --update; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/update-content-command.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/update-content-command.sh index 72b2ac9f7..b1840e552 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/update-content-command.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/update-content-command.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash if test -z "${SKIP_RAPIDS_BUILD_UTILS_UPDATE_CONTENT_COMMAND:-}"; then rapids-generate-scripts; From e162d06571398b9109589be25c0646137f78c280 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 13:05:16 -0800 Subject: [PATCH 031/217] add docstrings --- .../bin/checkout-same-branch.sh | 20 ++++++- .../bin/get-num-archs-jobs-and-load.sh | 51 ++++++++++++------ .../opt/rapids-build-utils/bin/list-repos.sh | 48 +++++++++++------ .../bin/make-conda-dependencies.sh | 45 ++++++++++------ .../rapids-build-utils/bin/make-conda-env.sh | 36 ++++++++----- .../bin/make-pip-dependencies.sh | 52 ++++++++++++------- .../rapids-build-utils/bin/make-pip-env.sh | 43 +++++++++------ .../bin/pull-repositories.sh | 28 ++++++++-- .../bin/push-repositories.sh | 28 ++++++++-- .../bin/python-conda-pkg-names.sh | 27 +++++++++- .../bin/python-pkg-names.sh | 21 +++++++- .../bin/python-pkg-roots.sh | 23 ++++++-- .../rapids-build-utils/bin/query-manifest.sh | 35 ++++++++----- 13 files changed, 334 insertions(+), 123 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/checkout-same-branch.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/checkout-same-branch.sh index df12dc930..1e51648e7 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/checkout-same-branch.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/checkout-same-branch.sh @@ -1,7 +1,21 @@ #!/usr/bin/env bash +# Usage: +# rapids-checkout-same-branch [OPTION]... +# +# Fetch all branches from the origin and upstream remotes, +# prompts the user to select a branch they all have in common. +# then checks out all repositories to this common branch. +# +# Boolean options: +# -h,--help,--usage print this text + +. devcontainer-utils-parse-args-from-docstring; + checkout_same_branch() { - set -euo pipefail; + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; eval "$( \ rapids-list-repos "$@" \ @@ -96,7 +110,9 @@ checkout_same_branch() { done; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"checkout-same-branch"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh index dc4554bc0..f231923ca 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh @@ -1,27 +1,42 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# rapids-get-num-archs-jobs-and-load [OPTION]... +# +# Compute an appropriate total number of jobs, load, and CUDA archs to build in parallel. +# This routine scales the input `-j` with respect to the `-a` and `-m` values, taking into account the +# amount of available system memory (free mem + swap), in order to balance the job and arch parallelism. +# +# note: This wouldn't be necessary if `nvcc` interacted with the POSIX jobserver. +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -a,--archs Build CUDA archs in parallel +# (default: 1) +# -j,--parallel Run parallel compilation jobs +# -m,--max-device-obj-memory-usage An upper-bound on the amount of memory each CUDA device object compilation +# is expected to take. This is used to estimate the number of parallel device +# object compilations that can be launched without hitting the system memory +# limit. +# Higher values yield fewer parallel CUDA device object compilations. +# (default: 1) + +. devcontainer-utils-parse-args-from-docstring; get_num_archs_jobs_and_load() { set -euo pipefail - local archs=""; - local parallel=""; - local max_device_obj_memory_usage=""; + parse_args_or_show_help - <<< "$@"; - eval "$( \ - devcontainer-utils-parse-args --names ' - a|archs | - j|parallel | - m|max-device-obj-memory-usage | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - archs="${a:-${archs:-}}"; + local archs="${a:-${archs:-}}"; archs="${archs//"true"/}"; - parallel="${j:-${parallel:-}}"; + + local parallel="${j:-${parallel:-}}"; parallel="${parallel//"true"/}"; - max_device_obj_memory_usage="${m:-${max_device_obj_memory_usage:-${MAX_DEVICE_OBJ_MEMORY_USAGE:-}}}"; + local max_device_obj_memory_usage="${m:-${max_device_obj_memory_usage:-${MAX_DEVICE_OBJ_MEMORY_USAGE:-}}}"; max_device_obj_memory_usage="${max_device_obj_memory_usage//"true"/}"; max_device_obj_memory_usage="${max_device_obj_memory_usage:-1}"; @@ -77,7 +92,9 @@ ____EOF echo "n_load=${n_load}"; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"get-num-archs-jobs-and-load"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh index 2703f1113..dad6e72c8 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh @@ -1,34 +1,52 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# rapids-list-repos [OPTION]... +# +# Query the `manifest.yaml` in a shell-friendly format, optionally filtering results by repository name. +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -r,--repo Filter the results to only include entries. +# (default: all repositories) + +. devcontainer-utils-parse-args-from-docstring; list_repos() { - set -euo pipefail; + set -Eeuo pipefail; - local repo=; + parse_args_or_show_help - <<< "$@"; - eval "$( \ - devcontainer-utils-parse-args --names ' - r|repo | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - repo="${r:-${repo:-}}"; + local repos=(); + repos+=(${r[@]:-}); unset r; + repos+=(${repo[@]:-}); unset repo; + repos=(${repos[@]:-}); local query=".repos | {repos: .}"; - if test -n "${repo:-}"; then + if test ${#repos[@]} -gt 0; then + # prefix each element + repos=("${repos[@]/#/'.name == "'}"); + # suffix each element + repos=("${repos[@]/%/'" or'}"); query="$(cat <<________EOF | tr -s '[:space:]' .repos - | map(select(.name == "${repo}")) + | map(select(${repos[@]} false)) | {repos: .} ________EOF )"; fi - rapids-query-manifest ${__rest__[@]} "${query}"; + rapids-query-manifest "$@" "${query}"; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"list-repos"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 90b4105b3..2a702ee68 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -1,4 +1,20 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# rapids-make-conda-dependencies [OPTION]... +# +# Generate a combined conda environment YAML file for all repos. +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -k,--key Only include the key(s) +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# --repo Only include dependencies for repo(s). + +. devcontainer-utils-parse-args-from-docstring; generate_env_yaml() { ( @@ -9,22 +25,19 @@ generate_env_yaml() { } make_conda_dependencies() { + set -Eeuo pipefail; - set -euo pipefail; + parse_args_or_show_help - <<< "$@"; local keys=(); + keys+=(${k[@]:-}); unset k; + keys+=(${key[@]:-}); unset key; + keys=(${keys[@]:-all}); - eval "$( \ - devcontainer-utils-parse-args --names ' - k|keys | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - if test -v k; then keys=(${k[@]}); fi - if test ${#keys[@]} -eq 0; then - keys=(all); - fi + local repos=(); + repos+=(${r[@]:-}); unset r; + repos+=(${repo[@]:-}); unset repo; + repos=(${repos[@]:-}); local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")"; @@ -35,7 +48,7 @@ make_conda_dependencies() { local conda_env_yamls=(); eval "$( \ - rapids-list-repos ${__rest__[@]} \ + rapids-list-repos ${repos[@]/#/-r } \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; @@ -94,7 +107,9 @@ make_conda_dependencies() { fi } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"make-conda-dependencies"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index 4d5e53aa2..c2855f404 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -1,20 +1,30 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# rapids-make-conda-env [OPTION]... +# +# Make a combined conda environment for all repos. +# +# Boolean options: +# -h,--help,--usage print this text +# -f,--force Delete the existing conda env and recreate it from scratch. +# +# Options that require values: +# -k,--key Only include the key(s) +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# --repo Only include dependencies for repo(s). + +. devcontainer-utils-parse-args-from-docstring; make_conda_env() { + set -Eeuo pipefail; - set -euo pipefail; + parse_args_or_show_help - <<< "$@"; - local force=; local env_name="${1}"; local env_file_name="${env_name}.yml"; - eval "$( \ - devcontainer-utils-parse-args --names ' - f|force | - ' - <<< "${@:2}" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - # Remove the current conda env if called with `-f|--force` if test -n "${f:-${force:-}}"; then rm -rf "$HOME/.conda/envs/${env_name}" \ @@ -24,7 +34,7 @@ make_conda_env() { local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.conda/envs/${env_file_name})"; - rapids-make-conda-dependencies ${__rest__[@]} > "${new_env_path}"; + rapids-make-conda-dependencies "$@" > "${new_env_path}"; if test -f "${new_env_path}"; then @@ -58,7 +68,9 @@ make_conda_env() { fi } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"make-conda-env"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index bdd775112..de1c21a90 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -1,4 +1,21 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# rapids-make-conda-dependencies [OPTION]... +# +# Generate a combined pip requirements file for all repos. +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -k,--key Only include the key(s) +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# --repo Only include dependencies for repo(s). +# -r,--requirement Path(s) to additional requirement files to include. + +. devcontainer-utils-parse-args-from-docstring; generate_requirements() { ( @@ -9,27 +26,22 @@ generate_requirements() { } make_pip_dependencies() { - set -Eeuo pipefail; - local keys=(); - local requirements=(); + parse_args_or_show_help - <<< "$@"; - eval "$( \ - devcontainer-utils-parse-args --names ' - k|keys | - r|requirement | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; + local keys=(); + keys+=(${k[@]:-}); unset k; + keys+=(${key[@]:-}); unset key; + keys=(${keys[@]:-py_build py_run py_test all}); - if test -v k; then keys=(${k[@]}); fi - if test ${#keys[@]} -eq 0; then - keys=(py_build py_run py_test all); - fi + local repos=(); + repos+=(${repo[@]:-}); unset repo; + repos=(${repos[@]:-}); - if test -v r; then requirements+=(${r[@]}); fi - if test -v requirement; then requirements+=(${requirement[@]}); fi + local requirements=(); + requirements+=(${r[@]:-}); unset r; + requirements+=(${requirement[@]:-}); unset requirement; local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; cuda_version="$(grep -o '^[0-9]*.[0-9]*' <<< "${cuda_version}")"; @@ -41,7 +53,7 @@ make_pip_dependencies() { local pip_reqs_txts=(); eval "$( \ - rapids-list-repos ${__rest__[@]} \ + rapids-list-repos ${repos[@]/#/-r } \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; @@ -113,7 +125,9 @@ make_pip_dependencies() { fi } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"make-pip-dependencies"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 397a90815..a72b311b8 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -1,20 +1,31 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# rapids-make-pip-env [OPTION]... +# +# Make a combined pip virtual environment for all repos. +# +# Boolean options: +# -h,--help,--usage print this text +# -f,--force Delete the existing pip venv and recreate it from scratch. +# +# Options that require values: +# -k,--key Only include the key(s) +# --repo Only include dependencies for repo(s). +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -r,--requirement Path(s) to additional requirement files to include. + +. devcontainer-utils-parse-args-from-docstring; make_pip_env() { + set -Eeuo pipefail; - set -euo pipefail; + parse_args_or_show_help - <<< "$@"; - local force=; local env_name="${1}"; local env_file_name="${env_name}.requirements.txt"; - eval "$( \ - devcontainer-utils-parse-args --names ' - f|force | - ' - <<< "${@:2}" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - # Remove the current virtual env if called with `-f|--force` if test -n "${f:-${force:-}}"; then rm -rf "$HOME/.local/share/venvs/${env_name}" \ @@ -24,10 +35,10 @@ make_pip_env() { local new_env_path="$(realpath -m /tmp/${env_file_name})"; local old_env_path="$(realpath -m ~/.local/share/venvs/${env_file_name})"; - rapids-make-pip-dependencies ${__rest__[@]} \ - | (grep -v -E '^$' || [ "$?" == "1" ]) \ - | tr -s "[:blank:]" \ - | LC_ALL=C sort -u \ + rapids-make-pip-dependencies "$@" \ + | (grep -v -E '^$' || [ "$?" == "1" ]) \ + | tr -s "[:blank:]" \ + | LC_ALL=C sort -u \ > "${new_env_path}" if test -f "${new_env_path}"; then @@ -66,7 +77,9 @@ make_pip_env() { fi } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"make-pip-env"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/pull-repositories.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/pull-repositories.sh index 049d392ee..0a05c415b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/pull-repositories.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/pull-repositories.sh @@ -1,7 +1,25 @@ #!/usr/bin/env bash -checkout_same_branch() { - set -euo pipefail; +# Usage: +# rapids-pull-repositories [OPTION]... +# +# Pull the latest commits from each repository's upstream remote. +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -r,--repo Filter the results to only include entries. +# (default: all repositories) + +. devcontainer-utils-parse-args-from-docstring; + +pull_repositories() { + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; eval "$( \ rapids-list-repos "$@" \ @@ -47,8 +65,10 @@ Please enter a branch name to pull (or leave empty to skip): " branch_name Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -r,--repo Filter the results to only include entries. +# (default: all repositories) + +. devcontainer-utils-parse-args-from-docstring; + +push_repositories() { + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; eval "$( \ rapids-list-repos "$@" \ @@ -24,8 +42,10 @@ checkout_same_branch() { done; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"push-repositories"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi -(checkout_same_branch "$@"); +(push_repositories "$@"); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-conda-pkg-names.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-conda-pkg-names.sh index 0ab279955..9cd1ba5e2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-conda-pkg-names.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-conda-pkg-names.sh @@ -1,11 +1,28 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +# Usage: +# rapids-python-conda-pkg-names [OPTION]... +# # Read the `name=""` fields from the meta.yaml files in each repository, # as determined by manifest.yaml. This seems to be the most reliable way to # determine the actual list of package names to exclude from the combined # conda env. +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -r,--repo Filter the results to only include entries. +# (default: all repositories) + +. devcontainer-utils-parse-args-from-docstring; + python_conda_pkg_names() { - set -euo pipefail; + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; eval "$( \ rapids-list-repos "$@" \ @@ -35,4 +52,10 @@ python_conda_pkg_names() { done } +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"python-conda-pkg-names"*}" ); then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + (python_conda_pkg_names "$@"); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-names.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-names.sh index dd61aede2..197dc3cc1 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-names.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-names.sh @@ -1,8 +1,25 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +# Usage: +# rapids-python-pkg-names [OPTION]... +# # List python package names as determined by manifest.yaml +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -r,--repo Filter the results to only include entries. +# (default: all repositories) + +. devcontainer-utils-parse-args-from-docstring; + python_pkg_names() { - set -euo pipefail; + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; eval "$( \ rapids-list-repos "$@" \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh index 00ffce50f..9ab763d95 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh @@ -1,8 +1,25 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# rapids-python-pkg-roots [OPTION]... +# +# List python package source dirs as determined by manifest.yaml +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -r,--repo Filter the results to only include entries. +# (default: all repositories) + +. devcontainer-utils-parse-args-from-docstring; -# List python package roots as determined by manifest.yaml python_pkg_roots() { - set -euo pipefail; + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; eval "$( \ rapids-list-repos "$@" \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/query-manifest.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/query-manifest.sh index e3cb0aabb..e2bb56172 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/query-manifest.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/query-manifest.sh @@ -1,18 +1,25 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# rapids-query-manifest [OPTION]... [QUERY] +# +# Query the `manifest.yaml` in a shell-friendly format. +# +# Boolean options: +# -h,--help,--usage print this text +# +# Options that require values: +# -m,--manifest Use a specific manifest.json +# (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) + +. devcontainer-utils-parse-args-from-docstring; query_manifest() { - set -euo pipefail; + set -Eeuo pipefail; - local manifest=; + parse_args_or_show_help - <<< "${@@Q}"; - eval "$( \ - devcontainer-utils-parse-args --names ' - m|manifest | - ' - <<< "${@@Q}" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - manifest="${m:-${manifest:-"${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}"}}"; + local manifest="${m:-${manifest:-"${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}"}}"; local query=; @@ -34,7 +41,7 @@ ________EOF )"; query="$(cat <<________EOF | tr -s '[:space:]' - ${__rest__[@]} + ${__rest__[-1]} | ${query} ________EOF )"; @@ -42,7 +49,9 @@ ________EOF yq -Mo json "${manifest}" | jq -r "${query}"; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"query-manifest"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi From d058b0510a8fb703eba8a8455e38b88c8836d7cb Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 13:05:37 -0800 Subject: [PATCH 032/217] narrow debug output by script name --- .../opt/rapids-build-utils/bin/make-vscode-workspace.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-vscode-workspace.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-vscode-workspace.sh index 559d281bc..232b7c622 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-vscode-workspace.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-vscode-workspace.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash get_repos_ordered() { local project_manifest_yml="${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}"; @@ -93,6 +93,12 @@ $(get_repos | with_git_dirs \ EOF } +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"make-vscode-workspace"*}" ); then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + if echo "$@" | grep -qE '(\-u|\-\-update)'; then (make_vscode_workspace "$@" > /tmp/workspace.code-workspace); if ! diff -BNqw ~/workspace.code-workspace /tmp/workspace.code-workspace >/dev/null 2>&1; then From 5cffd0d2fbfdbd371fbdc49faf9479037db93030 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 13:07:43 -0800 Subject: [PATCH 033/217] don't downcase C++ lib names, generate completions --- .../bin/generate-scripts.sh | 388 ++++++++++-------- 1 file changed, 210 insertions(+), 178 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index 672ca498a..31dac8843 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -1,47 +1,56 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash -TMPL=/opt/rapids-build-utils/bin/tmpl; - -TMP_SCRIPT_DIR=/tmp/rapids-build-utils -mkdir -p $TMP_SCRIPT_DIR +ALT_SCRIPT_DIR="${ALT_SCRIPT_DIR:-/usr/bin}"; +TEMPLATES="${TEMPLATES:-/opt/rapids-build-utils/bin/tmpl}"; +TMP_SCRIPT_DIR="${TMP_SCRIPT_DIR:-/tmp/rapids-build-utils}"; +COMPLETION_TMPL="${COMPLETION_TMPL:-"$(which devcontainer-utils-bash-completion.tmpl)"}"; clean_scripts() { set -euo pipefail; - for x in $(find $TMP_SCRIPT_DIR -maxdepth 1 -type f -printf '%f\n'); do - (sudo update-alternatives --remove-all $x > /dev/null 2>&1); - done - sudo rm -rf "$TMP_SCRIPT_DIR"; - mkdir -p $TMP_SCRIPT_DIR + mkdir -p "${TMP_SCRIPT_DIR}"; + find "${TMP_SCRIPT_DIR}"/ -maxdepth 1 -type f -exec basename {} \; \ + | xargs -r -I% sudo rm -f "${TMP_SCRIPT_DIR}"/% /etc/bash_completion.d/%; } generate_script() { local bin="${1:-}"; - if test -n "$bin" && ! test -f "$TMP_SCRIPT_DIR/${bin}"; then + if test -n "${bin}"; then ( - cat - \ - | envsubst '$NAME - $SRC_PATH - $PY_ENV - $PY_SRC - $PY_LIB - $CPP_LIB - $CPP_SRC - $CPP_ARGS - $CPP_DEPS - $GIT_TAG - $GIT_REPO - $GIT_HOST - $GIT_UPSTREAM - $PY_CMAKE_ARGS - $PIP_WHEEL_ARGS - $PIP_INSTALL_ARGS' \ - | sudo tee "$TMP_SCRIPT_DIR/${bin}" >/dev/null; - - sudo chmod +x "$TMP_SCRIPT_DIR/${bin}"; - - sudo update-alternatives --install \ - "/usr/bin/${bin}" "${bin}" "$TMP_SCRIPT_DIR/${bin}" 0 \ - >/dev/null 2>&1; + cat - \ + | envsubst '$NAME + $SRC_PATH + $PY_ENV + $PY_SRC + $PY_LIB + $CPP_LIB + $CPP_SRC + $CPP_ARGS + $CPP_DEPS + $GIT_TAG + $GIT_REPO + $GIT_HOST + $GIT_UPSTREAM + $PY_CMAKE_ARGS + $PIP_WHEEL_ARGS + $PIP_INSTALL_ARGS' \ + | tee "${TMP_SCRIPT_DIR}/${bin}" >/dev/null; + + chmod +x "${TMP_SCRIPT_DIR}/${bin}"; + + sudo ln -sf "${TMP_SCRIPT_DIR}/${bin}" "${ALT_SCRIPT_DIR}/${bin}"; + + if [[ "${bin}" != "${bin,,}" ]]; then + sudo ln -sf "${TMP_SCRIPT_DIR}/${bin,,}" "${ALT_SCRIPT_DIR}/${bin,,}"; + fi + + if test -f "${COMPLETION_TMPL}"; then + cat "${COMPLETION_TMPL}" \ + | SCRIPT="${bin}" \ + NAME="${bin//-/_}" \ + envsubst '$SCRIPT $NAME' \ + | sudo tee "/etc/bash_completion.d/${bin}" >/dev/null; + fi + ) & true; echo "$!" @@ -49,19 +58,25 @@ generate_script() { } generate_all_script_impl() { - local bin="$SCRIPT-all"; - if test -n "$bin" && ! test -f "$TMP_SCRIPT_DIR/${bin}"; then + local bin="${SCRIPT}-all"; + if test -n "${bin}" && ! test -f "${TMP_SCRIPT_DIR}/${bin}"; then ( - cat - \ - | envsubst '$NAMES - $SCRIPT' \ - | sudo tee "$TMP_SCRIPT_DIR/${bin}" >/dev/null; - - sudo chmod +x "$TMP_SCRIPT_DIR/${bin}"; - - sudo update-alternatives --install \ - "/usr/bin/${bin}" "${bin}" "$TMP_SCRIPT_DIR/${bin}" 0 \ - >/dev/null 2>&1; + cat - \ + | envsubst '$NAMES + $SCRIPT' \ + | tee "${TMP_SCRIPT_DIR}/${bin}" >/dev/null; + + chmod +x "${TMP_SCRIPT_DIR}/${bin}"; + + sudo ln -sf "${TMP_SCRIPT_DIR}/${bin}" "${ALT_SCRIPT_DIR}/${bin}"; + + if test -f "${COMPLETION_TMPL}"; then + cat "${COMPLETION_TMPL}" \ + | SCRIPT="${bin}" \ + NAME="${bin//-/_}" \ + envsubst '$SCRIPT $NAME' \ + | sudo tee "/etc/bash_completion.d/${bin}" >/dev/null; + fi ) & true; echo "$!" @@ -69,49 +84,59 @@ generate_all_script_impl() { } generate_all_script() { - ( - cat ${TMPL}/all.tmpl.sh \ - | generate_all_script_impl; + if test -f "${TEMPLATES}/all.tmpl.sh"; then ( + cat "${TEMPLATES}/all.tmpl.sh" \ + | generate_all_script_impl ; ) || true; + fi } generate_clone_script() { - ( - cat ${TMPL}/clone.tmpl.sh \ - | generate_script "clone-${NAME}"; + if test -f "${TEMPLATES}/clone.tmpl.sh"; then ( + cat "${TEMPLATES}/clone.tmpl.sh" \ + | generate_script "clone-${NAME}" ; ) || true; + fi } generate_repo_scripts() { local script_name; - for script_name in "configure" "build" "clean"; do ( - cat ${TMPL}/${script_name}.tmpl.sh \ - | generate_script "${script_name}-${NAME}"; - ) || true; + for script_name in "configure" "build" "clean"; do + if test -f "${TEMPLATES}/${script_name}.tmpl.sh"; then ( + cat "${TEMPLATES}/${script_name}.tmpl.sh" \ + | generate_script "${script_name}-${NAME}" ; + ) || true; + fi done } generate_cpp_scripts() { local script_name; - for script_name in "configure" "build" "clean" "install" "cpack"; do ( - cat ${TMPL}/cpp-${script_name}.tmpl.sh \ - | CPP_SRC="${SRC_PATH:-}${CPP_SRC:+/$CPP_SRC}" \ - generate_script "${script_name}-${CPP_LIB}-cpp"; - ) || true; + for script_name in "clean" "configure" "build" "cpack" "install"; do + if test -f "${TEMPLATES}/cpp-${script_name}.tmpl.sh"; then ( + cat "${TEMPLATES}/cpp-${script_name}.tmpl.sh" \ + | CPP_SRC="${SRC_PATH:-}${CPP_SRC:+/$CPP_SRC}" \ + generate_script "${script_name}-${CPP_LIB}-cpp"; + ) || true; + fi done } generate_python_scripts() { local script_name; - for script_name in "build" "clean"; do ( - cat ${TMPL}/python-${script_name}.tmpl.sh \ - | generate_script "${script_name}-${PY_LIB}-python"; - ) || true; + for script_name in "build" "clean"; do + if test -f "${TEMPLATES}/python-${script_name}.tmpl.sh"; then ( + cat "${TEMPLATES}/python-${script_name}.tmpl.sh" \ + | generate_script "${script_name}-${PY_LIB}-python"; + ) || true; + fi done - for script_name in "editable" "wheel"; do ( - cat ${TMPL}/python-build-${script_name}.tmpl.sh \ - | generate_script "build-${PY_LIB}-python-${script_name}"; - ) || true; + for script_name in "editable" "wheel"; do + if test -f "${TEMPLATES}/python-build-${script_name}.tmpl.sh"; then ( + cat "${TEMPLATES}/python-build-${script_name}.tmpl.sh" \ + | generate_script "build-${PY_LIB}-python-${script_name}"; + ) || true; + fi done } @@ -124,8 +149,6 @@ generate_scripts() { # Ensure we're in this script's directory cd "$( cd "$( dirname "$(realpath -m "${BASH_SOURCE[0]}")" )" && pwd )"; - # PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; - eval "$( \ rapids-list-repos "$@" \ | xargs -r -d'\n' -I% echo -n local %\; \ @@ -151,121 +174,126 @@ generate_scripts() { local git_tag="${repo}_git_tag"; local git_upstream="${repo}_git_upstream"; - repo_name="$(tr "[:upper:]" "[:lower:]" <<< "${!repo_name:-}")"; + repo_name="${!repo_name,,}"; + repo_names+=("${repo_name}"); - repo_names+=($repo_name); + local cpp_libs=(); + local cpp_dirs=(); - # Generate a clone script for each repo - ( - NAME="${repo_name:-}" \ - SRC_PATH=~/"${!repo_path:-}" \ - GIT_TAG="${!git_tag:-}" \ - GIT_REPO="${!git_repo:-}" \ - GIT_HOST="${!git_host:-}" \ - GIT_UPSTREAM="${!git_upstream:-}" \ - generate_clone_script ; - ) || true; + local py_libs=() + local py_dirs=() - if [[ -d ~/"${!repo_path:-}/.git" ]]; then + for ((j=0; j < ${!cpp_length:-0}; j+=1)); do - local cpp_libs=(); - local cpp_dirs=(); - - for ((j=0; j < ${!cpp_length:-0}; j+=1)); do - - local cpp_name="${repo}_cpp_${j}_name"; - local cpp_args="${repo}_cpp_${j}_args"; - local cpp_sub_dir="${repo}_cpp_${j}_sub_dir"; - local cpp_depends_length="${repo}_cpp_${j}_depends_length"; - local cpp_path=~/"${!repo_path:-}${!cpp_sub_dir:+/${!cpp_sub_dir}}"; - - cpp_dirs+=("${cpp_path}"); - cpp_libs+=("${!cpp_name:-}"); - cpp_name="$(tr "[:upper:]" "[:lower:]" <<< "${!cpp_name:-}")"; - - cpp_name_to_path["${cpp_name:-}"]="${cpp_path}"; - - local deps=(); - - for ((k=0; k < ${!cpp_depends_length:-0}; k+=1)); do - local dep="${repo}_cpp_${j}_depends_${k}"; - local dep_cpp_name=$(tr "[:upper:]" "[:lower:]" <<< "${!dep}"); - if ! test -v cpp_name_to_path["${dep_cpp_name}"]; then - continue; - fi - local dep_cpp_path="${cpp_name_to_path["${dep_cpp_name}"]}"; - - deps+=(-D${!dep}_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); - deps+=(-D$(tr "[:upper:]" "[:lower:]" <<< "${!dep}")_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); - deps+=(-D$(tr "[:lower:]" "[:upper:]" <<< "${!dep}")_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); - done - - ( - SRC_PATH=~/"${!repo_path:-}" \ - CPP_LIB="${cpp_name:-}" \ - CPP_SRC="${!cpp_sub_dir:-}" \ - CPP_ARGS="${!cpp_args:-}" \ - CPP_DEPS="${deps[@]}" \ - generate_cpp_scripts ; - ) || true; - done + local cpp_name="${repo}_cpp_${j}_name"; + local cpp_args="${repo}_cpp_${j}_args"; + local cpp_sub_dir="${repo}_cpp_${j}_sub_dir"; + local cpp_depends_length="${repo}_cpp_${j}_depends_length"; + local cpp_path=~/"${!repo_path:-}${!cpp_sub_dir:+/${!cpp_sub_dir}}"; - local args=(); - local deps=(); + cpp_dirs+=("${cpp_path}"); + cpp_libs+=("${!cpp_name:-}"); + cpp_name="${!cpp_name:-}"; + cpp_name_lower="${cpp_name,,}"; - for ((k=0; k < ${#cpp_libs[@]}; k+=1)); do - # Define both lowercase and uppercase - # `-DFIND__CPP=ON` and `-DFIND__CPP=ON` because the RAPIDS - # scikit-build CMakeLists.txt's aren't 100% consistent in the casing - local cpp_dir="${cpp_dirs[$k]}"; - local cpp_lib="${cpp_libs[$k]}"; - args+=(-DFIND_${cpp_lib}_CPP=ON); - args+=(-DFIND_$(tr "[:upper:]" "[:lower:]" <<< "${cpp_lib}")_CPP=ON); - args+=(-DFIND_$(tr "[:lower:]" "[:upper:]" <<< "${cpp_lib}")_CPP=ON); - deps+=(-D${cpp_lib}_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); - deps+=(-D$(tr "[:upper:]" "[:lower:]" <<< "${cpp_lib}")_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); - deps+=(-D$(tr "[:lower:]" "[:upper:]" <<< "${cpp_lib}")_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); - done + cpp_name_to_path["${cpp_name}"]="${cpp_path}"; - local py_libs=() - local py_dirs=() - - for ((j=0; j < ${!py_length:-0}; j+=1)); do - local py_env="${repo}_python_${j}_env"; - local py_name="${repo}_python_${j}_name"; - local py_cmake_args="${repo}_python_${j}_args_cmake"; - local pip_wheel_args="${repo}_python_${j}_args_wheel"; - local pip_install_args="${repo}_python_${j}_args_install"; - local py_sub_dir="${repo}_python_${j}_sub_dir"; - local py_depends_length="${repo}_python_${j}_depends_length"; - local py_path=~/"${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}"; - - py_libs+=(${!py_name}); - - ( - PY_SRC="${py_path}" \ - PY_LIB="${!py_name}" \ - PY_ENV="${!py_env:-}" \ - CPP_ARGS="${args[@]}" \ - CPP_DEPS="${deps[@]}" \ - PY_CMAKE_ARGS="${!py_cmake_args:-}" \ - PIP_WHEEL_ARGS="${!pip_wheel_args:-}" \ - PIP_INSTALL_ARGS="${!pip_install_args:-}" \ - generate_python_scripts ; - ) || true; - done; - - for ((k=0; k < ${#cpp_libs[@]}; k+=1)); do - cpp_libs[$k]="$(tr "[:upper:]" "[:lower:]" <<< "${cpp_libs[$k]}")"; + local deps=(); + + for ((k=0; k < ${!cpp_depends_length:-0}; k+=1)); do + local dep="${repo}_cpp_${j}_depends_${k}"; + local dep_cpp_name="${!dep}"; + if ! test -v cpp_name_to_path["${dep_cpp_name}"]; then + continue; + fi + local dep_cpp_path="${cpp_name_to_path["${dep_cpp_name}"]}"; + + deps+=(-D${!dep}_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); + deps+=(-D${!dep,,}_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); + deps+=(-D${!dep^^}_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); done + if [[ -d ~/"${!repo_path:-}/.git" ]]; then + ( + SRC_PATH=~/"${!repo_path:-}" \ + CPP_LIB="${cpp_name:-}" \ + CPP_SRC="${!cpp_sub_dir:-}" \ + CPP_ARGS="${!cpp_args:-}" \ + CPP_DEPS="${deps[@]}" \ + generate_cpp_scripts ; + ) || true; + fi + done + + local args=(); + local deps=(); + + for ((k=0; k < ${#cpp_libs[@]}; k+=1)); do + # Define both lowercase and uppercase + # `-DFIND__CPP=ON` and `-DFIND__CPP=ON` because the RAPIDS + # scikit-build CMakeLists.txt's aren't 100% consistent in the casing + local cpp_dir="${cpp_dirs[$k]}"; + local cpp_lib="${cpp_libs[$k]}"; + args+=(-DFIND_${cpp_lib}_CPP=ON); + args+=(-DFIND_${cpp_lib,,}_CPP=ON); + args+=(-DFIND_${cpp_lib^^}_CPP=ON); + deps+=(-D${cpp_lib}_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); + deps+=(-D${cpp_lib,,}_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); + deps+=(-D${cpp_lib^^}_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); + done + + for ((j=0; j < ${!py_length:-0}; j+=1)); do + local py_env="${repo}_python_${j}_env"; + local py_name="${repo}_python_${j}_name"; + local py_cmake_args="${repo}_python_${j}_args_cmake"; + local pip_wheel_args="${repo}_python_${j}_args_wheel"; + local pip_install_args="${repo}_python_${j}_args_install"; + local py_sub_dir="${repo}_python_${j}_sub_dir"; + local py_depends_length="${repo}_python_${j}_depends_length"; + local py_path=~/"${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}"; + + py_dirs+=("${py_path}"); + py_libs+=("${!py_name}"); + + if [[ -d ~/"${!repo_path:-}/.git" ]]; then ( - NAME="${repo_name:-}" \ - PY_LIB="${py_libs[@]}" \ - CPP_LIB="${cpp_libs[@]}" \ - generate_repo_scripts ; + SRC_PATH=~/"${!repo_path:-}" \ + PY_SRC="${py_path}" \ + PY_LIB="${!py_name}" \ + PY_ENV="${!py_env:-}" \ + CPP_ARGS="${args[@]}" \ + CPP_DEPS="${deps[@]}" \ + PY_CMAKE_ARGS="${!py_cmake_args:-}" \ + PIP_WHEEL_ARGS="${!pip_wheel_args:-}" \ + PIP_INSTALL_ARGS="${!pip_install_args:-}" \ + generate_python_scripts ; ) || true; + fi + done; + + if [[ -d ~/"${!repo_path:-}/.git" ]]; then + ( + NAME="${repo_name:-}" \ + PY_LIB="${py_libs[@]}" \ + CPP_LIB="${cpp_libs[@]}" \ + generate_repo_scripts ; + ) || true; fi + + # Generate a clone script for each repo + ( + NAME="${repo_name:-}" \ + SRC_PATH=~/"${!repo_path:-}" \ + PY_LIB="${py_libs[@]}" \ + PY_SRC="${py_dirs[@]}" \ + CPP_LIB="${cpp_libs[@]}" \ + CPP_SRC="${cpp_dirs[@]}" \ + GIT_TAG="${!git_tag:-}" \ + GIT_REPO="${!git_repo:-}" \ + GIT_HOST="${!git_host:-}" \ + GIT_UPSTREAM="${!git_upstream:-}" \ + generate_clone_script ; + ) || true; done sudo find /opt/rapids-build-utils \ @@ -274,7 +302,7 @@ generate_scripts() { unset cpp_name_to_path; - for script in "clone" "clean" "configure" "build"; do + for script in "clone" "clean" "configure" "build" "cpack" "install"; do # Generate a script to run a script for all repos ( NAMES="${repo_names[@]}" \ @@ -284,11 +312,15 @@ generate_scripts() { done; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"generate-scripts"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi -(clean_scripts); +clean_scripts; + +rm -rf "${TMP_SCRIPT_DIR}"/*; for pid in $(generate_scripts "$@"); do while [[ -e "/proc/$pid" ]]; do From 9c9340db0261af764ee45fec439dd77f0b0a1e0e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 13:12:31 -0800 Subject: [PATCH 034/217] add docstrings --- .../rapids-build-utils/bin/tmpl/all.tmpl.sh | 23 ++- .../rapids-build-utils/bin/tmpl/build.tmpl.sh | 36 +++-- .../rapids-build-utils/bin/tmpl/clean.tmpl.sh | 21 ++- .../rapids-build-utils/bin/tmpl/clone.tmpl.sh | 62 ++++---- .../bin/tmpl/configure.tmpl.sh | 36 ++++- .../bin/tmpl/cpp-build.tmpl.sh | 49 ++++--- .../bin/tmpl/cpp-clean.tmpl.sh | 23 ++- .../bin/tmpl/cpp-configure.tmpl.sh | 65 +++++---- .../bin/tmpl/cpp-cpack.tmpl.sh | 39 ++++-- .../bin/tmpl/cpp-install.tmpl.sh | 24 +++- .../bin/tmpl/python-build-editable.tmpl.sh | 84 +++++------ .../bin/tmpl/python-build-wheel.tmpl.sh | 132 ++++++++++-------- .../bin/tmpl/python-build.tmpl.sh | 34 +++-- .../bin/tmpl/python-clean.tmpl.sh | 33 +++-- 14 files changed, 427 insertions(+), 234 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.tmpl.sh index 8e33cec52..f5ac0d392 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.tmpl.sh @@ -1,5 +1,22 @@ +#!/usr/bin/env bash + +# Usage: +# ${SCRIPT}-all [OPTION]... +# +# Runs the ${SCRIPT} in each ${NAMES}. +# +# Forwards all arguments to each underlying script. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output + +. devcontainer-utils-parse-args-from-docstring; + ${SCRIPT}_all() { - set -euo pipefail; + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; for name in ${NAMES}; do if type ${SCRIPT}-${name} >/dev/null 2>&1; then @@ -8,7 +25,9 @@ ${SCRIPT}_all() { done } -if [[ -n "${rapids_build_utils_debug:-}" ]]; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"${SCRIPT}-all"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh index 9cc2fdec4..2aab58c90 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh @@ -1,33 +1,41 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash -build_${NAME}() { - set -euo pipefail; +# Usage: +# build-${PY_LIB} [OPTION]... +# +# Configure and build ${CPP_LIB}, then build an editable install or wheel of ${PY_LIB}. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# +# Options that require values: +# -t,--type editable|wheel The type of Python build to run (editable or wheel) +# (default: editable) - local mode="editable"; +. devcontainer-utils-parse-args-from-docstring; - # pull out the --mode editable|wheel arg (if any) - eval "$( \ - devcontainer-utils-parse-args --names ' - m|mode | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; +build_${NAME}() { + set -Eeuo pipefail; - mode="${m:-${mode:-"editable"}}"; + parse_args_or_show_help - <<< "$@"; for lib in ${CPP_LIB}; do if type build-${lib}-cpp >/dev/null 2>&1; then build-${lib}-cpp ${__rest__[@]}; fi done + for lib in ${PY_LIB}; do if type build-${lib}-python >/dev/null 2>&1; then - build-${lib}-python -m ${mode} ${__rest__[@]}; + build-${lib}-python --type ${t:-${type:-"editable"}} ${__rest__[@]}; fi done } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"build-${NAME}"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clean.tmpl.sh index 611222178..ed66235e0 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clean.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clean.tmpl.sh @@ -1,7 +1,20 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# clean-${NAME} [OPTION]... +# +# Clean the ${NAME} repository. +# +# Boolean options: +# -h,--help,--usage print this text + +. devcontainer-utils-parse-args-from-docstring; clean_${NAME}() { - set -euo pipefail; + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; + for lib in ${CPP_LIB}; do if type clean-${lib}-cpp >/dev/null 2>&1; then clean-${lib}-cpp "$@"; @@ -14,7 +27,9 @@ clean_${NAME}() { done } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"clean-${NAME}"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh index 8e569a9be..d31e234ab 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh @@ -1,32 +1,42 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash -clone_${NAME}() { - set -euo pipefail; +# Usage: +# clone-${NAME}-cpp [OPTION]... +# +# Clone the ${NAME} repository. +# +# Boolean options: +# -h,--help,--usage print this text +# --no-fork don't prompt the user to fork the repo if a user fork isn't found +# (default: false) +# --no-update-env don't update the Python env with the repo's dependencies after cloning +# (default: false) +# --clone-upstream always clone the upstream, not the user's fork +# (default: false) +# +# Options that require values: +# -b,--branch check the repo out to +# (default: `${NAME}.git.tag` in manifest.yaml) +# -d,--directory clone the repo into +# (default: `${NAME}.path` in manifest.yaml) +# -u,--upstream set as the `upstream` remote +# (default: `${NAME}.git.upstream` in manifest.yaml) - if [[ ! -d "${SRC_PATH}"/.git ]]; then +. devcontainer-utils-parse-args-from-docstring; - local branch=; - local upstream=; - local directory=; - local no_fork=; - local clone_upstream=; - local no_update_env=; +clone_${NAME}() { + set -Eeuo pipefail; - eval "$( \ - devcontainer-utils-parse-args --names ' - b|branch | - d|directory | - u|upstream | - no-fork | - clone-upstream | - no-update-env | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; + parse_args_or_show_help - <<< "$@"; + + if [[ ! -d "${SRC_PATH}"/.git ]]; then - branch="${b:-${branch:-"${GIT_TAG}"}}"; - directory="${d:-${directory:-"${SRC_PATH}"}}"; - upstream="${u:-${upstream:-"${GIT_UPSTREAM}/${GIT_REPO}"}}"; + local no_fork="${no_fork:-}"; + local no_update_env="${no_update_env:-}"; + local clone_upstream="${clone_upstream:-}"; + local branch="${b:-${branch:-"${GIT_TAG}"}}"; + local directory="${d:-${directory:-"${SRC_PATH}"}}"; + local upstream="${u:-${upstream:-"${GIT_UPSTREAM}/${GIT_REPO}"}}"; echo 'Cloning ${NAME}' 1>&2; @@ -59,7 +69,9 @@ clone_${NAME}() { fi } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"clone-${NAME}"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/configure.tmpl.sh index 8062c3346..b0af00960 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/configure.tmpl.sh @@ -1,7 +1,35 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# configure-${CPP_LIB} [OPTION]... +# +# Runs the configure script in each ${CPP_LIB}. +# +# Forwards all arguments to each underlying script. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# +# Options that require values: +# -a,--archs Build CUDA archs in parallel +# (default: 1) +# -j,--parallel Run parallel compilation jobs +# -m,--max-device-obj-memory-usage An upper-bound on the amount of memory each CUDA device object compilation +# is expected to take. This is used to estimate the number of parallel device +# object compilations that can be launched without hitting the system memory +# limit. +# Higher values yield fewer parallel CUDA device object compilations. +# (default: 1) +# -D* [:]= Create or update a cmake cache entry. + +. devcontainer-utils-parse-args-from-docstring; configure_${NAME}() { - set -euo pipefail; + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; + for lib in ${CPP_LIB}; do if type configure-${lib}-cpp >/dev/null 2>&1; then configure-${lib}-cpp "$@"; @@ -9,7 +37,9 @@ configure_${NAME}() { done } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"configure-${NAME}"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh index 4d0def9be..931e22ba1 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh @@ -1,29 +1,44 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# build-${CPP_LIB}-cpp [OPTION]... +# +# Configure and build ${CPP_LIB}. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# +# Options that require values: +# -a,--archs Build CUDA archs in parallel +# (default: 1) +# -j,--parallel Run parallel compilation jobs +# -m,--max-device-obj-memory-usage An upper-bound on the amount of memory each CUDA device object compilation +# is expected to take. This is used to estimate the number of parallel device +# object compilations that can be launched without hitting the system memory +# limit. +# Higher values yield fewer parallel CUDA device object compilations. +# (default: 1) +# -D* [:]= Create or update a cmake cache entry. + +. devcontainer-utils-parse-args-from-docstring; build_${CPP_LIB}_cpp() { - set -Eeuo pipefail; + parse_args_or_show_help - <<< "$@"; + if [[ ! -d "${CPP_SRC}" ]]; then exit 1; fi - local verbose=""; - - eval "$( \ - devcontainer-utils-parse-args --names ' - v|verbose | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - verbose="${v:-${verbose:-}}"; + local verbose="${v:-${verbose:-}}"; configure-${CPP_LIB}-cpp ${verbose:+-v} ${__rest__[@]}; - eval "$( \ - rapids-get-num-archs-jobs-and-load ${__rest__[@]} \ - | xargs -r -d'\n' -I% echo -n local %\; \ + eval "$( \ + rapids-get-num-archs-jobs-and-load "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ )"; # Build C++ lib @@ -38,7 +53,9 @@ build_${CPP_LIB}_cpp() { ) 2>&1; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"build-${CPP_LIB}-cpp"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh index f25c83766..b86d39846 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh @@ -1,14 +1,31 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# clean-${CPP_LIB}-cpp [OPTION]... +# +# Clean the ${CPP_LIB} build dirs. +# +# Boolean options: +# -h,--help,--usage print this text + +. devcontainer-utils-parse-args-from-docstring; clean_${CPP_LIB}_cpp() { + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; - set -euo pipefail; + if [[ ! -d "${CPP_SRC}" ]]; then + exit 1; + fi rm -rf "${CPP_SRC}/build/latest"/* \ "${CPP_SRC}"/compile_commands.json; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"clean-${CPP_LIB}-cpp"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh index 9b53e5e1f..7f195c349 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh @@ -1,41 +1,42 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# configure-${CPP_LIB}-cpp [OPTION]... +# +# Configure ${CPP_LIB}. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# +# Options that require values: +# -a,--archs Build CUDA archs in parallel +# (default: 1) +# -j,--parallel Run parallel compilation jobs +# -m,--max-device-obj-memory-usage An upper-bound on the amount of memory each CUDA device object compilation +# is expected to take. This is used to estimate the number of parallel device +# object compilations that can be launched without hitting the system memory +# limit. +# Higher values yield fewer parallel CUDA device object compilations. +# (default: 1) +# -D* [:]= Create or update a cmake cache entry. + +. devcontainer-utils-parse-args-from-docstring; configure_${CPP_LIB}_cpp() { - set -Eeuo pipefail; + parse_args_or_show_help - <<< "$@"; + if [[ ! -d "${CPP_SRC}" ]]; then exit 1; fi - local archs=""; - local parallel=""; - local max_device_obj_memory_usage=""; - - local verbose=""; - - eval "$( \ - devcontainer-utils-parse-args --names ' - a|archs | - j|parallel | - m|max-device-obj-memory-usage | - v|verbose | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - verbose="${v:-${verbose:-}}"; - - archs="${a:-${archs:-}}"; - parallel="${j:-${parallel:-}}"; - max_device_obj_memory_usage="${m:-${max_device_obj_memory_usage:-}}"; + local verbose="${v:-${verbose:-}}"; - eval "$( \ - rapids-get-num-archs-jobs-and-load \ - ${archs:+-a ${archs}} \ - ${parallel:+-j ${parallel}} \ - ${max_device_obj_memory_usage:+-m ${max_device_obj_memory_usage}} \ - | xargs -r -d'\n' -I% echo -n local %\; \ + eval "$( \ + rapids-get-num-archs-jobs-and-load "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ )"; local build_type="$(rapids-parse-cmake-build-type ${__rest__[@]} | tr '[:upper:]' '[:lower:]')"; @@ -60,7 +61,7 @@ configure_${CPP_LIB}_cpp() { cmake_args+=(${__rest__[@]}); time ( - CUDAFLAGS="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}" \ + CUDAFLAGS="${CUDAFLAGS:+$CUDAFLAGS }-t${n_arch}" \ cmake ${cmake_args[@]}; { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} configure time:"; ) 2>&1; @@ -75,7 +76,9 @@ configure_${CPP_LIB}_cpp() { fi } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"configure-${CPP_LIB}-cpp"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh index ee6909acb..bdf78fb37 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-cpack.tmpl.sh @@ -1,21 +1,27 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# cpack-${CPP_LIB}-cpp [OPTION]... +# +# CPack ${CPP_LIB}. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# +# Options that require values: +# -o,--out-dir copy cpack'd TGZ file into +# (default: none) + +. devcontainer-utils-parse-args-from-docstring; cpack_${CPP_LIB}_cpp() { - set -Eeuo pipefail; - local verbose=""; - - eval "$( \ - devcontainer-utils-parse-args --names ' - v|verbose | - o|out-dir | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; + parse_args_or_show_help - <<< "$@"; - verbose="${v:-${verbose:-}}"; - out_dir="${o:-${out_dir:-}}"; + local verbose="${v:-${verbose:-}}"; + local out_dir="${o:-${out_dir:-}}"; time ( ( @@ -26,14 +32,17 @@ cpack_${CPP_LIB}_cpp() { if test -d "${out_dir}"/; then find "${CPP_SRC}"/build/latest/ \ -iname "${CPP_LIB}-*.tar.gz" \ - -exec cp -a "{}" "${out_dir}"/ \; + -print0 \ + | xargs -0 -I% cp -a "%" "${out_dir}"/; fi { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} CPack time:"; ) 2>&1; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"cpack-${CPP_LIB}-cpp"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh index bee264bb4..121cf9f1e 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-install.tmpl.sh @@ -1,16 +1,34 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash -install_${CPP_LIB}_cpp() { +# Usage: +# install-${CPP_LIB}-cpp [OPTION]... +# +# Install ${CPP_LIB}. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# +# Options that require values: +# -p,--prefix install C++ library into +# (default: none) + +. devcontainer-utils-parse-args-from-docstring; +install_${CPP_LIB}_cpp() { set -Eeuo pipefail; + parse_args_or_show_help - <<< "$@"; + time ( cmake --install "${CPP_SRC}"/build/latest/ "$@"; { set +x; } 2>/dev/null; echo -n "lib${CPP_LIB} install time:"; ) 2>&1; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"install-${CPP_LIB}-cpp"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh index 228aca4b2..e44267b39 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-editable.tmpl.sh @@ -1,42 +1,41 @@ -#! /usr/bin/env bash - -build_${PY_LIB}_python_inplace() { - +#!/usr/bin/env bash + +# Usage: +# build-${PY_LIB}-python-editable [OPTION]... +# +# Build ${PY_LIB} in editable mode. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# +# Options that require values: +# -a,--archs Build CUDA archs in parallel +# (default: 1) +# -j,--parallel Run parallel compilation jobs +# -m,--max-device-obj-memory-usage An upper-bound on the amount of memory each CUDA device object compilation +# is expected to take. This is used to estimate the number of parallel device +# object compilations that can be launched without hitting the system memory +# limit. +# Higher values yield fewer parallel CUDA device object compilations. +# (default: 1) + +. devcontainer-utils-parse-args-from-docstring; + +build_${PY_LIB}_python_editable() { set -Eeuo pipefail; + parse_args_or_show_help - <<< "$@"; + if [[ ! -d "${PY_SRC}" ]]; then exit 1; fi - local archs=""; - local parallel=""; - local max_device_obj_memory_usage=""; + local verbose="${v:-${verbose:-}}"; - local verbose=""; - - eval "$( \ - devcontainer-utils-parse-args --names ' - a|archs | - j|parallel | - m|max-device-obj-memory-usage | - v|verbose | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - verbose="${v:-${verbose:-}}"; - config_settings="${C:-${config_settings:-}}"; - - archs="${a:-${archs:-}}"; - parallel="${j:-${parallel:-}}"; - max_device_obj_memory_usage="${m:-${max_device_obj_memory_usage:-}}"; - - eval "$( \ - rapids-get-num-archs-jobs-and-load \ - ${archs:+-a ${archs}} \ - ${parallel:+-j ${parallel}} \ - ${max_device_obj_memory_usage:+-m ${max_device_obj_memory_usage}} \ - | xargs -r -d'\n' -I% echo -n local %\; \ + eval "$( \ + rapids-get-num-archs-jobs-and-load "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ )"; local cmake_args=(${PY_CMAKE_ARGS}); @@ -48,8 +47,11 @@ build_${PY_LIB}_python_inplace() { cmake_args+=(${__rest__[@]}); local ninja_args=(); + local pip_args=(${PIP_INSTALL_ARGS}); + if test -n "${verbose}"; then ninja_args+=("-v"); + pip_args+=("-vv"); fi if test -n "${n_jobs}"; then @@ -60,12 +62,6 @@ build_${PY_LIB}_python_inplace() { ninja_args+=("-l${n_load}"); fi - local pip_args=(${PIP_INSTALL_ARGS}); - - if test -n "${verbose}"; then - pip_args+=("-vv"); - fi - pip_args+=("--no-build-isolation"); pip_args+=("--no-deps"); pip_args+=("--editable"); @@ -78,11 +74,12 @@ build_${PY_LIB}_python_inplace() { time ( export ${PY_ENV} PATH="$PATH"; - local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; - local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; + local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t${n_arch}"; + local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t${n_arch}"; CUDAFLAGS="${cudaflags}" \ CMAKE_GENERATOR="Ninja" \ + PARALLEL_LEVEL="${n_jobs}" \ CMAKE_ARGS="${cmake_args[@]}" \ SKBUILD_BUILD_OPTIONS="${ninja_args[@]}" \ NVCC_APPEND_FLAGS="${nvcc_append_flags}" \ @@ -93,8 +90,11 @@ build_${PY_LIB}_python_inplace() { ) 2>&1; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"build-${PY_LIB}-python"*}" \ + || test -z "${rapids_build_utils_debug##*"build-${PY_LIB}-python-editable"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi -build_${PY_LIB}_python_inplace "$@"; +build_${PY_LIB}_python_editable "$@"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh index 4e7a15470..6ed634982 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build-wheel.tmpl.sh @@ -1,52 +1,64 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash + +# Usage: +# build-${PY_LIB}-python-wheel [OPTION]... +# +# Build a ${PY_LIB} wheel. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# --no-clean Don't clean up build directories. +# --no-deps Don't install package dependencies. +# --no-verify Don't verify if built wheel is valid. +# --no-cache-dir Disable the pip cache. +# --use-pep517 Use PEP 517 for building source distributions. +# --no-use-pep517 Don't use PEP 517 for building source distributions +# --no-build-isolation Disable isolation when building a modern source distribution. Build +# dependencies specified by PEP 518 must be already installed if this option is +# used. +# --pre Include pre-release and development versions. By default, pip only finds +# stable versions. +# --prefer-binary Prefer binary packages over source packages, even if the source packages are +# newer. +# +# Options that require values: +# -a,--archs Build CUDA archs in parallel +# (default: 1) +# -j,--parallel Run parallel compilation jobs +# -m,--max-device-obj-memory-usage An upper-bound on the amount of memory each CUDA device object compilation +# is expected to take. This is used to estimate the number of parallel device +# object compilations that can be launched without hitting the system memory +# limit. +# Higher values yield fewer parallel CUDA device object compilations. +# (default: 1) +# --cache-dir Store the cache data in . +# --ignore-requires-python Ignore the Requires-Python information. +# --only-binary Do not use source packages. Can be supplied multiple times, and each time +# adds to the existing value. Accepts either ":all:" to disable all source +# packages, ":none:" to empty the set, or one or more package names with commas +# between them. Packages without binary distributions will fail to install when +# this option is used on them. +# -w,--wheel-dir copy built wheel into +# (default: none) + +. devcontainer-utils-parse-args-from-docstring; build_${PY_LIB}_python_wheel() { - set -Eeuo pipefail; + parse_args_or_show_help - <<< "$@"; + if [[ ! -d "${PY_SRC}" ]]; then exit 1; fi - local archs=""; - local parallel=""; - local max_device_obj_memory_usage=""; - - local verbose=""; - local wheel_dir=""; - - eval "$( \ - devcontainer-utils-parse-args --names ' - a|archs | - j|parallel | - m|max-device-obj-memory-usage | - v|verbose | - w|wheel-dir | - prefer-binary | - only-binary | - no-deps | - no-clean | - no-verify | - no-cache-dir | - no-use-pep517 | - no-build-isolation | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; - - verbose="${v:-${verbose:-}}"; - wheel_dir="${w:-${wheel_dir:-}}"; + local verbose="${v:-${verbose:-}}"; + local wheel_dir="${w:-${wheel_dir:-}}"; - archs="${a:-${archs:-}}"; - parallel="${j:-${parallel:-}}"; - max_device_obj_memory_usage="${m:-${max_device_obj_memory_usage:-}}"; - - eval "$( \ - rapids-get-num-archs-jobs-and-load \ - ${archs:+-a ${archs}} \ - ${parallel:+-j ${parallel}} \ - ${max_device_obj_memory_usage:+-m ${max_device_obj_memory_usage}} \ - | xargs -r -d'\n' -I% echo -n local %\; \ + eval "$( \ + rapids-get-num-archs-jobs-and-load "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ )"; local cmake_args=(${PY_CMAKE_ARGS}); @@ -62,9 +74,11 @@ build_${PY_LIB}_python_wheel() { cmake_args+=(${__rest__[@]}); local ninja_args=(); + local pip_args=(${PIP_WHEEL_ARGS}); if test -n "${verbose}"; then ninja_args+=("-v"); + pip_args+=("-vv"); fi if test -n "${n_jobs}"; then @@ -75,27 +89,22 @@ build_${PY_LIB}_python_wheel() { ninja_args+=("-l${n_load}"); fi - local pip_args=(${PIP_WHEEL_ARGS}); - - if test -n "${verbose}"; then - pip_args+=("-vv"); + if test -n "${pre:-}"; then + pip_args+=("--pre"); fi if test -n "${wheel_dir:-}"; then - pip_args+=("-w" "${wheel_dir}"); + pip_args+=("--wheel-dir" "${wheel_dir}"); fi if test -n "${prefer_binary:-}"; then pip_args+=("--prefer-binary"); - if [ "${prefer_binary:-}" != "true" ]; then - pip_args+=("${prefer_binary:-}"); - fi fi - if test -n "${only_binary:-}"; then + if test -n "${only_binary[@]:-}"; then pip_args+=("--only-binary"); - if [ "${only_binary:-}" != "true" ]; then - pip_args+=("${only_binary:-}"); + if [ "${only_binary[@]}" != "true" ]; then + pip_args+=("${only_binary[@]}"); fi fi @@ -111,11 +120,15 @@ build_${PY_LIB}_python_wheel() { pip_args+=("--no-verify"); fi - if test -n "${no_cache_dir:-}"; then + if test -n "${cache_dir:-}"; then + pip_args+=("--cache-dir" "${cache_dir:-}"); + elif test -n "${no_cache_dir:-}"; then pip_args+=("--no-cache-dir"); fi - if test -n "${no_use_pep517:-}"; then + if test -n "${use_pep517:-}"; then + pip_args+=("--use-pep517"); + elif test -n "${no_use_pep517:-}"; then pip_args+=("--no-use-pep517"); fi @@ -123,6 +136,10 @@ build_${PY_LIB}_python_wheel() { pip_args+=("--no-build-isolation"); fi + if test -n "${ignore_requires_python:-}"; then + pip_args+=("--ignore-requires-python"); + fi + pip_args+=("${PY_SRC}"); trap "rm -rf '${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT; @@ -130,8 +147,8 @@ build_${PY_LIB}_python_wheel() { time ( export ${PY_ENV} PATH="$PATH"; - local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t=${n_arch}"; - local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=${n_arch}"; + local cudaflags="${CUDAFLAGS:+$CUDAFLAGS }-t${n_arch}"; + local nvcc_append_flags="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t${n_arch}"; CUDAFLAGS="${cudaflags}" \ CMAKE_GENERATOR="Ninja" \ @@ -145,7 +162,10 @@ build_${PY_LIB}_python_wheel() { ) 2>&1; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"build-${PY_LIB}-python"*}" \ + || test -z "${rapids_build_utils_debug##*"build-${PY_LIB}-python-wheel"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh index 5326023c5..579e23118 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh @@ -1,23 +1,31 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash -build_${PY_LIB}_python() { +# Usage: +# build-${PY_LIB}-python [OPTION]... +# +# Build an editable install or wheel of ${PY_LIB}. +# +# Boolean options: +# -h,--help,--usage print this text +# -v,--verbose verbose output +# +# Options that require values: +# -t,--type editable|wheel The type of Python build to run (editable or wheel) +# (default: editable) - local mode="editable"; +. devcontainer-utils-parse-args-from-docstring; - # pull out the --mode editable|wheel arg (if any) - eval "$( \ - devcontainer-utils-parse-args --names ' - m|mode | - ' - <<< "$@" \ - | xargs -r -d'\n' -I% echo -n local %\; \ - )"; +build_${PY_LIB}_python() { + set -Eeuo pipefail; - mode="${m:-${mode:-"editable"}}"; + parse_args_or_show_help - <<< "$@"; - build-${PY_LIB}-python-${mode} ${__rest__[@]}; + build-${PY_LIB}-python-${t:-${type:-"editable"}} ${__rest__[@]}; } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"build-${PY_LIB}-python"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh index 22d7f0bee..ed8279d6c 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh @@ -1,8 +1,23 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash -clean_${PY_LIB}_cpp() { +# Usage: +# clean-${PY_LIB}-python [OPTION]... +# +# Clean the ${PY_LIB} build dirs. +# +# Boolean options: +# -h,--help,--usage print this text - set -euo pipefail; +. devcontainer-utils-parse-args-from-docstring; + +clean_${PY_LIB}_python() { + set -Eeuo pipefail; + + parse_args_or_show_help - <<< "$@"; + + if [[ ! -d "${PY_SRC}" ]]; then + exit 1; + fi local py_lib="${PY_LIB}"; @@ -22,17 +37,19 @@ clean_${PY_LIB}_cpp() { if test -d "${PY_SRC}"/build; then local slug="$(uname -s)-$(uname -m)"; - rm -rf \ - `# scikit-buld-core build dirs` \ + rm -rf \ + `# scikit-buld-core build dirs` \ "${PY_SRC}"/build/cp{${py_ver},${py_ver/./}}-cp{${py_ver},${py_ver/./}}* \ - `# setuptools/distutils build dirs` \ + `# setuptools/distutils build dirs` \ "${PY_SRC}"/build/{lib,temp,dist,bdist}.${slug,,} \ "${PY_SRC}"/build/{lib,temp,dist,bdist}.${slug,,}-{,cpython}{,-}{${py_ver},${py_ver/./}}; fi } -if test -n "${rapids_build_utils_debug:-}"; then +if test -n "${rapids_build_utils_debug:-}" \ +&& ( test -z "${rapids_build_utils_debug##*"all"*}" \ + || test -z "${rapids_build_utils_debug##*"clean-${PY_LIB}-python"*}" ); then PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; fi -clean_${PY_LIB}_cpp "$@"; +clean_${PY_LIB}_python "$@"; From 58493006a37a9c1cebd25fe4353087d4de45587d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 13:38:08 -0800 Subject: [PATCH 035/217] add --omit to rapids-list-repos --- .../opt/rapids-build-utils/bin/list-repos.sh | 29 ++++++++++++++----- .../bin/make-conda-dependencies.sh | 3 ++ .../rapids-build-utils/bin/make-conda-env.sh | 3 ++ .../bin/make-pip-dependencies.sh | 3 ++ .../rapids-build-utils/bin/make-pip-env.sh | 5 +++- .../bin/pull-repositories.sh | 2 ++ .../bin/push-repositories.sh | 2 ++ .../bin/python-conda-pkg-names.sh | 2 ++ .../bin/python-pkg-names.sh | 2 ++ .../bin/python-pkg-roots.sh | 2 ++ 10 files changed, 45 insertions(+), 8 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh index dad6e72c8..3c04ffb1c 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh @@ -11,7 +11,9 @@ # Options that require values: # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) -# -r,--repo Filter the results to only include entries. +# -o,--omit Filter the results to exclude entries. +# (default: none) +# -r,--repo Filter the results to include entries. # (default: all repositories) . devcontainer-utils-parse-args-from-docstring; @@ -26,19 +28,32 @@ list_repos() { repos+=(${repo[@]:-}); unset repo; repos=(${repos[@]:-}); + local omits=(); + omits+=(${o[@]:-}); unset o; + omits+=(${omit[@]:-}); unset omit; + omits=(${omits[@]:-}); + local query=".repos | {repos: .}"; + local filters=(); if test ${#repos[@]} -gt 0; then # prefix each element repos=("${repos[@]/#/'.name == "'}"); # suffix each element repos=("${repos[@]/%/'" or'}"); - query="$(cat <<________EOF | tr -s '[:space:]' - .repos - | map(select(${repos[@]} false)) - | {repos: .} -________EOF -)"; + filters+=("| map(select(${repos[@]} false))"); + fi + + if test ${#omits[@]} -gt 0; then + # prefix each element + omits=("${omits[@]/#/'.name != "'}"); + # suffix each element + omits=("${omits[@]/%/'" and'}"); + filters+=("| map(select(${omits[@]} true))"); + fi + + if test -n "${filters:-}"; then + query=".repos ${filters[@]} | {repos: .}"; fi rapids-query-manifest "$@" "${query}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 2a702ee68..a1ef52bf9 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -12,7 +12,10 @@ # -k,--key Only include the key(s) # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Omit dependencies for repo(s). +# (default: none) # --repo Only include dependencies for repo(s). +# (default: all repositories) . devcontainer-utils-parse-args-from-docstring; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index c2855f404..0bef98333 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -13,7 +13,10 @@ # -k,--key Only include the key(s) # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Omit dependencies for repo(s). +# (default: none) # --repo Only include dependencies for repo(s). +# (default: all repositories) . devcontainer-utils-parse-args-from-docstring; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index de1c21a90..bbd367987 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -12,7 +12,10 @@ # -k,--key Only include the key(s) # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Omit dependencies for repo(s). +# (default: none) # --repo Only include dependencies for repo(s). +# (default: all repositories) # -r,--requirement Path(s) to additional requirement files to include. . devcontainer-utils-parse-args-from-docstring; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index a72b311b8..296950040 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -11,9 +11,12 @@ # # Options that require values: # -k,--key Only include the key(s) -# --repo Only include dependencies for repo(s). # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Omit dependencies for repo(s). +# (default: none) +# --repo Only include dependencies for repo(s). +# (default: all repositories) # -r,--requirement Path(s) to additional requirement files to include. . devcontainer-utils-parse-args-from-docstring; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/pull-repositories.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/pull-repositories.sh index 0a05c415b..5f5be46a1 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/pull-repositories.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/pull-repositories.sh @@ -11,6 +11,8 @@ # Options that require values: # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Filter the results to exclude entries. +# (default: none) # -r,--repo Filter the results to only include entries. # (default: all repositories) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/push-repositories.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/push-repositories.sh index 5130eabd9..739dcdf11 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/push-repositories.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/push-repositories.sh @@ -11,6 +11,8 @@ # Options that require values: # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Filter the results to exclude entries. +# (default: none) # -r,--repo Filter the results to only include entries. # (default: all repositories) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-conda-pkg-names.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-conda-pkg-names.sh index 9cd1ba5e2..9bdcdad5b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-conda-pkg-names.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-conda-pkg-names.sh @@ -14,6 +14,8 @@ # Options that require values: # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Filter the results to exclude entries. +# (default: none) # -r,--repo Filter the results to only include entries. # (default: all repositories) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-names.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-names.sh index 197dc3cc1..567cc3660 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-names.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-names.sh @@ -11,6 +11,8 @@ # Options that require values: # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Filter the results to exclude entries. +# (default: none) # -r,--repo Filter the results to only include entries. # (default: all repositories) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh index 9ab763d95..524aabf47 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh @@ -11,6 +11,8 @@ # Options that require values: # -m,--manifest Use a specific manifest.json # (default: ${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}) +# -o,--omit Filter the results to exclude entries. +# (default: none) # -r,--repo Filter the results to only include entries. # (default: all repositories) From 5055b15d61c5155e826652872606d402340c7d53 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 13:39:37 -0800 Subject: [PATCH 036/217] install the utils repo from disk, ignore cron.d startup error --- .devcontainer/cuda11.8-conda/devcontainer.json | 2 ++ .devcontainer/cuda11.8-pip/devcontainer.json | 2 ++ .devcontainer/cuda12.0-conda/devcontainer.json | 2 ++ .devcontainer/cuda12.0-pip/devcontainer.json | 2 ++ features/src/utils/devcontainer-feature.json | 2 +- 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 5aa82c5f4..5643f3426 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -10,9 +10,11 @@ }, "hostRequirements": {"gpu": "optional"}, "features": { + "./features/src/utils": {}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ + "./features/src/utils", "./features/src/rapids-build-utils" ], "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 94212b758..a66f82a99 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -12,11 +12,13 @@ "features": { "./features/src/ucx": {"version": "1.14.1"}, "./features/src/cuda": {"installcuDNN": true}, + "./features/src/utils": {}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ "./features/src/ucx", "./features/src/cuda", + "./features/src/utils", "./features/src/rapids-build-utils" ], "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json index 40c6b8899..ae02ba519 100644 --- a/.devcontainer/cuda12.0-conda/devcontainer.json +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -10,9 +10,11 @@ }, "hostRequirements": {"gpu": "optional"}, "features": { + "./features/src/utils": {}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ + "./features/src/utils", "./features/src/rapids-build-utils" ], "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json index 186498d18..d7465e4f8 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -12,11 +12,13 @@ "features": { "./features/src/ucx": {"version": "1.14.1"}, "./features/src/cuda": {"installcuDNN": true}, + "./features/src/utils": {}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ "./features/src/ucx", "./features/src/cuda", + "./features/src/utils", "./features/src/rapids-build-utils" ], "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index f0f3fb3de..40820dba5 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -7,7 +7,7 @@ "BASH_ENV": "/etc/bash.bash_env" }, "postCreateCommand": [ - "cron" + "/bin/bash", "-c", "if ! test -f /var/run/crond.pid; then cron 2>/dev/null || true; fi" ], "postAttachCommand": [ "/bin/bash", From 33f6b0c494744122efab19e0af130da7f7f6fe0c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 14:04:51 -0800 Subject: [PATCH 037/217] parse args after env name --- .../opt/rapids-build-utils/bin/make-conda-env.sh | 2 +- .../opt/rapids-build-utils/bin/make-pip-env.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index 0bef98333..94bf97521 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -23,7 +23,7 @@ make_conda_env() { set -Eeuo pipefail; - parse_args_or_show_help - <<< "$@"; + parse_args_or_show_help - <<< "${@:2}"; local env_name="${1}"; local env_file_name="${env_name}.yml"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 296950040..563af8829 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -24,7 +24,7 @@ make_pip_env() { set -Eeuo pipefail; - parse_args_or_show_help - <<< "$@"; + parse_args_or_show_help - <<< "${@:2}"; local env_name="${1}"; local env_file_name="${env_name}.requirements.txt"; From dcee2a6b107c36ea039f01e9d70c60d75ef3db4b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 14:20:54 -0800 Subject: [PATCH 038/217] fix typo --- features/src/utils/opt/devcontainer/bin/parse-args.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/features/src/utils/opt/devcontainer/bin/parse-args.sh b/features/src/utils/opt/devcontainer/bin/parse-args.sh index 74f2732b1..645248f03 100755 --- a/features/src/utils/opt/devcontainer/bin/parse-args.sh +++ b/features/src/utils/opt/devcontainer/bin/parse-args.sh @@ -51,7 +51,7 @@ parse_args() { shift; # -fooval local found=""; - if test "${#vars_array}" -gt 0 \ + if test "${#vars_array[@]}" -gt 0 \ && grep -qP "^--?${vars}([^\s])+$" <<< "${arg:-}"; then for name in ${vars_array[@]}; do if grep -qP "^--?${name}([0-9]|\.)+$" <<< "${arg:-}"; then @@ -78,8 +78,6 @@ parse_args() { else val="true"; fi - else - rest+=("${arg@Q}"); fi else rest+=("${@@Q}"); From 9bee84a0d125f89c7d40a2c979264250c03f4226 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 14:46:18 -0800 Subject: [PATCH 039/217] fix paths --- .../opt/rapids-build-utils/bin/generate-scripts.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index 31dac8843..8f51e376f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -208,9 +208,9 @@ generate_scripts() { fi local dep_cpp_path="${cpp_name_to_path["${dep_cpp_name}"]}"; - deps+=(-D${!dep}_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); - deps+=(-D${!dep,,}_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); - deps+=(-D${!dep^^}_ROOT=\"$(realpath -m ~/${dep_cpp_path}/build/latest)\"); + deps+=(-D${!dep}_ROOT=\"${dep_cpp_path}/build/latest\"); + deps+=(-D${!dep,,}_ROOT=\"${dep_cpp_path}/build/latest\"); + deps+=(-D${!dep^^}_ROOT=\"${dep_cpp_path}/build/latest\"); done if [[ -d ~/"${!repo_path:-}/.git" ]]; then @@ -237,9 +237,9 @@ generate_scripts() { args+=(-DFIND_${cpp_lib}_CPP=ON); args+=(-DFIND_${cpp_lib,,}_CPP=ON); args+=(-DFIND_${cpp_lib^^}_CPP=ON); - deps+=(-D${cpp_lib}_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); - deps+=(-D${cpp_lib,,}_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); - deps+=(-D${cpp_lib^^}_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); + deps+=(-D${cpp_lib}_ROOT=\"${cpp_dir}/build/latest\"); + deps+=(-D${cpp_lib,,}_ROOT=\"${cpp_dir}/build/latest\"); + deps+=(-D${cpp_lib^^}_ROOT=\"${cpp_dir}/build/latest\"); done for ((j=0; j < ${!py_length:-0}; j+=1)); do From 6f82337e087692c7e0acda7f189e29365d8560ec Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 22 Jan 2024 16:50:41 -0800 Subject: [PATCH 040/217] make individual