From 696b0dd472d42c7e71c992bd2e3def2d40fadebb Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Wed, 7 Jun 2023 19:20:39 -0400 Subject: [PATCH] Publish docs as github artifacts during CI This PR saves library docs as github artifacts so they can be easily viewed for review. Discussed in --- .github/workflows/ci.yml | 36 ++++++++++++++++ .../docker/host-x86_64/mingw-check/Dockerfile | 3 ++ src/ci/github-actions/ci.yml | 16 +++++++ src/ci/scripts/create-doc-artifacts.sh | 42 +++++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100755 src/ci/scripts/create-doc-artifacts.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 210ec72a11e0b..ecc30901c7a7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,7 @@ jobs: env: CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" SCCACHE_BUCKET: rust-lang-ci-sccache2 TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" CACHE_DOMAIN: ci-caches.rust-lang.org @@ -143,6 +144,17 @@ jobs: AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" if: success() && !env.SKIP_JOB + - name: create github artifacts + run: src/ci/scripts/create-doc-artifacts.sh + if: success() && !env.SKIP_JOB + - name: upload artifacts to github + uses: actions/upload-artifact@v3 + with: + name: "${{ env.DOC_ARTIFACT_NAME }}" + path: obj/artifacts/doc + if-no-files-found: ignore + retention-days: 5 + if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh env: @@ -156,6 +168,7 @@ jobs: env: CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" SCCACHE_BUCKET: rust-lang-ci-sccache2 DEPLOY_BUCKET: rust-lang-ci2 TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" @@ -567,6 +580,17 @@ jobs: AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" if: success() && !env.SKIP_JOB + - name: create github artifacts + run: src/ci/scripts/create-doc-artifacts.sh + if: success() && !env.SKIP_JOB + - name: upload artifacts to github + uses: actions/upload-artifact@v3 + with: + name: "${{ env.DOC_ARTIFACT_NAME }}" + path: obj/artifacts/doc + if-no-files-found: ignore + retention-days: 5 + if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh env: @@ -581,6 +605,7 @@ jobs: DIST_TRY_BUILD: 1 CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" SCCACHE_BUCKET: rust-lang-ci-sccache2 DEPLOY_BUCKET: rust-lang-ci2 TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" @@ -682,6 +707,17 @@ jobs: AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" if: success() && !env.SKIP_JOB + - name: create github artifacts + run: src/ci/scripts/create-doc-artifacts.sh + if: success() && !env.SKIP_JOB + - name: upload artifacts to github + uses: actions/upload-artifact@v3 + with: + name: "${{ env.DOC_ARTIFACT_NAME }}" + path: obj/artifacts/doc + if-no-files-found: ignore + retention-days: 5 + if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh env: diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index 515890aef8df8..85a9a5d33752f 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -45,6 +45,9 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \ python3 ../x.py test --stage 0 src/tools/compiletest && \ python3 ../x.py test --stage 0 core alloc std test proc_macro && \ # Build both public and internal documentation. + RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library && \ + mkdir -p /checkout/obj/staging/doc && \ + cp -r build/x86_64-unknown-linux-gnu/doc /checkout/obj/staging && \ RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 compiler && \ RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library/test && \ /scripts/validate-toolstate.sh && \ diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index d3cb6b6ed5220..f8986bbc108c5 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -34,6 +34,8 @@ x--expand-yaml-anchors--remove: - &shared-ci-variables CI_JOB_NAME: ${{ matrix.name }} CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - &public-variables SCCACHE_BUCKET: rust-lang-ci-sccache2 @@ -229,6 +231,20 @@ x--expand-yaml-anchors--remove: TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} <<: *step + - name: create github artifacts + run: src/ci/scripts/create-doc-artifacts.sh + <<: *step + + - name: upload artifacts to github + uses: actions/upload-artifact@v3 + with: + # name is set in previous step + name: ${{ env.DOC_ARTIFACT_NAME }} + path: obj/artifacts/doc + if-no-files-found: ignore + retention-days: 5 + <<: *step + - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh env: diff --git a/src/ci/scripts/create-doc-artifacts.sh b/src/ci/scripts/create-doc-artifacts.sh new file mode 100755 index 0000000000000..2516b0d850594 --- /dev/null +++ b/src/ci/scripts/create-doc-artifacts.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Compress doc artifacts and name them based on the commit, or the date if +# commit is not available. + +set -euox pipefail + +# Try to get short commit hash, fallback to date +if [ -n "$HEAD_SHA" ]; then + short_rev=$(echo "${HEAD_SHA}" | cut -c1-8) +else + short_rev=$(git rev-parse --short HEAD || date -u +'%Y-%m-%dT%H%M%SZ') +fi + +# Try to get branch, fallback to none +branch=$(git branch --show-current || echo) + +if [ -n "$branch" ]; then + branch="${branch}-" +fi + +if [ "${GITHUB_EVENT_NAME:=none}" = "pull_request" ]; then + pr_num=$(echo "$GITHUB_REF_NAME" | cut -d'/' -f1) + name="doc-${pr_num}-${short_rev}" +else + name="doc-${branch}${short_rev}" +fi + + +if [ -d "obj/staging/doc" ]; then + mkdir -p obj/artifacts/doc + + # Level 12 seems to give a good tradeoff of time vs. space savings + ZSTD_CLEVEL=12 ZSTD_NBTHREADS=4 \ + tar --zstd -cf "obj/artifacts/doc/${name}.tar.zst" -C obj/staging/doc . + + ls -lh obj/artifacts/doc +fi + +# Set this environment variable for future use if running in CI +if [ -n "$GITHUB_ENV" ]; then + echo "DOC_ARTIFACT_NAME=${name}" >> "$GITHUB_ENV" +fi