diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 9cad7f90c90..1c91a1443ef 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -181,67 +181,71 @@ publish-wheels-to-s3: tags: ["arch:amd64"] image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 stage: package - needs: - - job: "ddtrace package" - artifacts: true + needs: [ "ddtrace package", "package version" ] variables: BUCKET: dd-trace-py-builds - script: - - set -euo pipefail - - shopt -s nullglob - # Find wheels - - WHEELS=(pywheels/*.whl) - - | - if [ ${#WHEELS[@]} -eq 0 ]; then - echo "No wheels found in pywheels/"; exit 1 - fi - - echo "Found ${#WHEELS[@]} wheel(s):" - - printf ' - %s\n' "${WHEELS[@]}" + script: | + set -euo pipefail + shopt -s nullglob - # Upload all wheels to commit prefix and pipeline-id prefix - - aws s3 cp --recursive --exclude "*" --include "*.whl" pywheels "s3://${BUCKET}/${CI_COMMIT_SHA}/" - - aws s3 cp --recursive --exclude "*" --include "*.whl" pywheels "s3://${BUCKET}/${CI_PIPELINE_ID}/" + # Find wheels + WHEELS=(pywheels/*.whl) + if [ ${#WHEELS[@]} -eq 0 ]; then echo "No wheels found in pywheels/"; exit 1; fi + echo "Found ${#WHEELS[@]} wheel(s):" + printf ' - %s\n' "${WHEELS[@]}" - - | - S3_BASE_COMMIT="https://${BUCKET}.s3.amazonaws.com/${CI_COMMIT_SHA}" - S3_BASE_PIPE="https://${BUCKET}.s3.amazonaws.com/${CI_PIPELINE_ID}" + # Upload wheels to S3 + aws s3 cp --recursive --exclude "*" --include "*.whl" pywheels "s3://${BUCKET}/${CI_COMMIT_SHA}/" + aws s3 cp --recursive --exclude "*" --include "*.whl" pywheels "s3://${BUCKET}/${CI_PIPELINE_ID}/" - generate_index_html() { - local outfile="$1" - { - echo "" - for w in "${WHEELS[@]}"; do - fname="$(basename "$w")" - enc_fname="${fname//+/%2B}" - echo "${fname}
" - done - echo "" - } > "${outfile}" - } + # Generate and upload helper scripts and index for commit + S3_BASE_COMMIT="https://${BUCKET}.s3.amazonaws.com/${CI_COMMIT_SHA}" + .gitlab/scripts/generate-s3-helper-scripts.sh "${S3_BASE_COMMIT}" "commit" + .gitlab/scripts/generate-index-html.sh "commit" + aws s3 cp "commit.download.sh" "s3://${BUCKET}/${CI_COMMIT_SHA}/download.sh" --content-type text/x-shellscript + aws s3 cp "commit.install.sh" "s3://${BUCKET}/${CI_COMMIT_SHA}/install.sh" --content-type text/x-shellscript + aws s3 cp "commit.index.html" "s3://${BUCKET}/${CI_COMMIT_SHA}/index.html" --content-type text/html - # Generate both minimal indexes - generate_index_html "index.commit.html" - generate_index_html "index.pipeline.html" + # Generate and upload helper scripts and index for pipeline + S3_BASE_PIPE="https://${BUCKET}.s3.amazonaws.com/${CI_PIPELINE_ID}" + .gitlab/scripts/generate-s3-helper-scripts.sh "${S3_BASE_PIPE}" "pipeline" + .gitlab/scripts/generate-index-html.sh "pipeline" + aws s3 cp "pipeline.download.sh" "s3://${BUCKET}/${CI_PIPELINE_ID}/download.sh" --content-type text/x-shellscript + aws s3 cp "pipeline.install.sh" "s3://${BUCKET}/${CI_PIPELINE_ID}/install.sh" --content-type text/x-shellscript + aws s3 cp "pipeline.index.html" "s3://${BUCKET}/${CI_PIPELINE_ID}/index.html" --content-type text/html - # Upload to each S3 prefix - aws s3 cp "index.commit.html" "s3://${BUCKET}/${CI_COMMIT_SHA}/index.html" --content-type text/html - aws s3 cp "index.pipeline.html" "s3://${BUCKET}/${CI_PIPELINE_ID}/index.html" --content-type text/html + # Branch uploads (if on main or release branch) + if [ "${CI_COMMIT_BRANCH:-}" = "main" ] || [[ "${CI_COMMIT_BRANCH:-}" =~ ^[0-9]+\.[0-9]+$ ]]; then + aws s3 cp --recursive --exclude "*" --include "*.whl" pywheels "s3://${BUCKET}/${CI_COMMIT_BRANCH}/" - # Print the clickable URLs - COMMIT_INDEX_URL="${S3_BASE_COMMIT}/index.html" - PIPE_INDEX_URL="${S3_BASE_PIPE}/index.html" - echo "S3 index (commit): ${COMMIT_INDEX_URL}" - echo "S3 index (pipeline): ${PIPE_INDEX_URL}" + S3_BASE_BRANCH="https://${BUCKET}.s3.amazonaws.com/${CI_COMMIT_BRANCH}" + .gitlab/scripts/generate-s3-helper-scripts.sh "${S3_BASE_BRANCH}" "branch" + .gitlab/scripts/generate-index-html.sh "branch" + aws s3 cp "branch.download.sh" "s3://${BUCKET}/${CI_COMMIT_BRANCH}/download.sh" --content-type text/x-shellscript + aws s3 cp "branch.install.sh" "s3://${BUCKET}/${CI_COMMIT_BRANCH}/install.sh" --content-type text/x-shellscript + aws s3 cp "branch.index.html" "s3://${BUCKET}/${CI_COMMIT_BRANCH}/index.html" --content-type text/html + fi - # Also publish to branch path if on main or a release branch - if [ "${CI_COMMIT_BRANCH:-}" = "main" ] || [[ "${CI_COMMIT_BRANCH:-}" =~ ^[0-9]+\.[0-9]+$ ]]; then - echo "Publishing to ${CI_COMMIT_BRANCH}/ path" - aws s3 cp --recursive --exclude "*" --include "*.whl" pywheels "s3://${BUCKET}/${CI_COMMIT_BRANCH}/" - generate_index_html "index.branch.html" - aws s3 cp "index.branch.html" "s3://${BUCKET}/${CI_COMMIT_BRANCH}/index.html" --content-type text/html - BRANCH_INDEX_URL="https://${BUCKET}.s3.amazonaws.com/${CI_COMMIT_BRANCH}/index.html" - echo "S3 index (branch): ${BRANCH_INDEX_URL}" - fi + # Print all URLs at the end (after noisy aws s3 cp commands) + echo "" + echo "=== Artifacts published to S3 ===" + echo "" + echo "Commit artifacts:" + echo " Index: ${S3_BASE_COMMIT}/index.html" + echo " Download: ${S3_BASE_COMMIT}/download.sh" + echo " Install: ${S3_BASE_COMMIT}/install.sh" + echo "" + echo "Pipeline artifacts:" + echo " Index: ${S3_BASE_PIPE}/index.html" + echo " Download: ${S3_BASE_PIPE}/download.sh" + echo " Install: ${S3_BASE_PIPE}/install.sh" + if [ "${CI_COMMIT_BRANCH:-}" = "main" ] || [[ "${CI_COMMIT_BRANCH:-}" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "" + echo "Branch artifacts (${CI_COMMIT_BRANCH}):" + echo " Index: ${S3_BASE_BRANCH}/index.html" + echo " Download: ${S3_BASE_BRANCH}/download.sh" + echo " Install: ${S3_BASE_BRANCH}/install.sh" + fi # Extract package version from pyproject.toml and save to dotenv artifact "package version": diff --git a/.gitlab/scripts/generate-index-html.sh b/.gitlab/scripts/generate-index-html.sh new file mode 100755 index 00000000000..0a946c305ab --- /dev/null +++ b/.gitlab/scripts/generate-index-html.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: generate-index-html.sh +# Example: generate-index-html.sh "commit" + +if [ $# -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +OUTPUT_PREFIX="$1" + +# Find all wheels +WHEELS=(pywheels/*.whl) +if [ ${#WHEELS[@]} -eq 0 ]; then + echo "ERROR: No wheels found in pywheels/" >&2 + exit 1 +fi + +# Generate index.html +{ + echo '' + for w in "${WHEELS[@]}"; do + fname="$(basename "$w")" + # URL-encode special characters (especially +) + enc_fname="${fname//+/%2B}" + echo "${fname}
" + done + echo "" +} > "${OUTPUT_PREFIX}.index.html" + +echo "Generated ${OUTPUT_PREFIX}.index.html" diff --git a/.gitlab/scripts/generate-s3-helper-scripts.sh b/.gitlab/scripts/generate-s3-helper-scripts.sh new file mode 100755 index 00000000000..bb79aac71eb --- /dev/null +++ b/.gitlab/scripts/generate-s3-helper-scripts.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: generate-s3-helper-scripts.sh +# Example: generate-s3-helper-scripts.sh "https://dd-trace-py-builds.s3.amazonaws.com/main" "main" + +if [ $# -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +S3_BASE_URL="$1" +OUTPUT_PREFIX="$2" + +if [ -z "${PACKAGE_VERSION:-}" ]; then + echo "Error: PACKAGE_VERSION environment variable is not set." >&2 + exit 1 +fi +echo "Detected version: ${PACKAGE_VERSION}" + +# Generate download.sh +cat > "${OUTPUT_PREFIX}.download.sh" << 'DOWNLOAD_SCRIPT_EOF' +#!/usr/bin/env bash +set -euo pipefail + +# Parse arguments +DEST_DIR="." +PIP_ARGS="" + +while [[ $# -gt 0 ]]; do + case $1 in + --dest) + DEST_DIR="$2" + shift 2 + ;; + --python-version) + PIP_ARGS="$PIP_ARGS --python-version $2" + shift 2 + ;; + --platform) + PIP_ARGS="$PIP_ARGS --platform $2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + echo "Usage: $0 [--dest DIR] [--python-version VERSION] [--platform PLATFORM]" >&2 + exit 1 + ;; + esac +done + +# Download wheel +echo "Downloading ddtrace==VERSION_PLACEHOLDER from S3_URL_PLACEHOLDER/index.html" +pip download --no-index --no-deps \ + --find-links S3_URL_PLACEHOLDER/index.html \ + ddtrace==VERSION_PLACEHOLDER \ + $PIP_ARGS \ + -d "${DEST_DIR}" + +echo "Downloaded to ${DEST_DIR}" +DOWNLOAD_SCRIPT_EOF + +# Replace placeholders in download.sh +sed -i.bak \ + -e "s|S3_URL_PLACEHOLDER|${S3_BASE_URL}|g" \ + -e "s|VERSION_PLACEHOLDER|${PACKAGE_VERSION}|g" \ + "${OUTPUT_PREFIX}.download.sh" +rm "${OUTPUT_PREFIX}.download.sh.bak" + +# Generate install.sh +cat > "${OUTPUT_PREFIX}.install.sh" << 'INSTALL_SCRIPT_EOF' +#!/usr/bin/env bash +set -euo pipefail + +# Parse arguments +PIP_ARGS="" + +while [[ $# -gt 0 ]]; do + case $1 in + --python-version) + PIP_ARGS="$PIP_ARGS --python-version $2" + shift 2 + ;; + --platform) + PIP_ARGS="$PIP_ARGS --platform $2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + echo "Usage: $0 [--python-version VERSION] [--platform PLATFORM]" >&2 + exit 1 + ;; + esac +done + +# Create temp directory and ensure cleanup +TMP_DIR=$(mktemp -d) +trap "rm -rf '${TMP_DIR}'" EXIT + +# Download and install +echo "Downloading ddtrace==VERSION_PLACEHOLDER from S3_URL_PLACEHOLDER/index.html" +pip download --no-index --no-deps \ + --find-links S3_URL_PLACEHOLDER/index.html \ + ddtrace==VERSION_PLACEHOLDER \ + $PIP_ARGS \ + -d "${TMP_DIR}" + +echo "Installing ddtrace==VERSION_PLACEHOLDER" +pip install "${TMP_DIR}"/ddtrace-*.whl + +echo "Successfully installed ddtrace==VERSION_PLACEHOLDER" +INSTALL_SCRIPT_EOF + +# Replace placeholders in install.sh +sed -i.bak \ + -e "s|S3_URL_PLACEHOLDER|${S3_BASE_URL}|g" \ + -e "s|VERSION_PLACEHOLDER|${PACKAGE_VERSION}|g" \ + "${OUTPUT_PREFIX}.install.sh" +rm "${OUTPUT_PREFIX}.install.sh.bak" + +echo "Generated ${OUTPUT_PREFIX}.download.sh and ${OUTPUT_PREFIX}.install.sh"