Skip to content

Commit

Permalink
Test
Browse files Browse the repository at this point in the history
  • Loading branch information
@ehuss
ehuss committed Oct 11, 2024
1 parent 51f3a70 commit 4bf068d
Show file tree
Hide file tree
Showing 3 changed files with 172 additions and 95 deletions.
155 changes: 75 additions & 80 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,6 @@
name: CI
on:
push:
branches:
- auto
- try
- try-perf
- automation/bors/try
pull_request:
branches:
- "**"
Expand Down Expand Up @@ -107,11 +102,11 @@ jobs:
# Rust Log Analyzer can't currently detect the PR number of a GitHub
# Actions build on its own, so a hint in the log message is needed to
# point it in the right direction.
- name: configure the PR in which the error message will be posted
run: echo "[CI_PR_NUMBER=$num]"
env:
num: ${{ github.event.number }}
if: needs.calculate_matrix.outputs.run_type == 'pr'
# - name: configure the PR in which the error message will be posted
# run: echo "[CI_PR_NUMBER=$num]"
# env:
# num: ${{ github.event.number }}
# if: needs.calculate_matrix.outputs.run_type == 'pr'

- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
Expand All @@ -122,8 +117,8 @@ jobs:
# which then uses log commands to actually set them.
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}

- name: ensure the channel matches the target branch
run: src/ci/scripts/verify-channel.sh
# - name: ensure the channel matches the target branch
# run: src/ci/scripts/verify-channel.sh

- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
Expand All @@ -146,23 +141,23 @@ jobs:
- name: install tidy
run: src/ci/scripts/install-tidy.sh

- name: install WIX
run: src/ci/scripts/install-wix.sh
# - name: install WIX
# run: src/ci/scripts/install-wix.sh

- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh

- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh

- name: install MinGW
run: src/ci/scripts/install-mingw.sh
# - name: install MinGW
# run: src/ci/scripts/install-mingw.sh

- name: install ninja
run: src/ci/scripts/install-ninja.sh

- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
# - name: enable ipv6 on Docker
# run: src/ci/scripts/enable-docker-ipv6.sh

# Disable automatic line ending conversion (again). On Windows, when we're
# installing dependencies, something switches the git configuration directory or
Expand All @@ -172,14 +167,14 @@ jobs:
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh

- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
# - name: ensure line endings are correct
# run: src/ci/scripts/verify-line-endings.sh

- name: ensure backported commits are in upstream branches
run: src/ci/scripts/verify-backported-commits.sh
# - name: ensure backported commits are in upstream branches
# run: src/ci/scripts/verify-backported-commits.sh

- name: ensure the stable version number is correct
run: src/ci/scripts/verify-stable-version-number.sh
# - name: ensure the stable version number is correct
# run: src/ci/scripts/verify-stable-version-number.sh

- name: run the build
# Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
Expand All @@ -188,63 +183,63 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}

- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh

- name: upload artifacts to github
uses: actions/upload-artifact@v4
with:
# name is set in previous step
name: ${{ env.DOC_ARTIFACT_NAME }}
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5

- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
# Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
# builders *should* have the AWS credentials available. Still, explicitly
# adding the condition is helpful as this way CI will not silently skip
# deploying artifacts from a dist builder if the variables are misconfigured,
# erroring about invalid credentials instead.
if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'

- name: upload job metrics to DataDog
if: needs.calculate_matrix.outputs.run_type != 'pr'
env:
DATADOG_SITE: datadoghq.com
DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
DD_GITHUB_JOB_NAME: ${{ matrix.name }}
run: |
npm install -g @datadog/datadog-ci@^2.x.x
python3 src/ci/scripts/upload-build-metrics.py build/cpu-usage.csv
# - name: create github artifacts
# run: src/ci/scripts/create-doc-artifacts.sh

# - name: upload artifacts to github
# uses: actions/upload-artifact@v4
# with:
# # name is set in previous step
# name: ${{ env.DOC_ARTIFACT_NAME }}
# path: obj/artifacts/doc
# if-no-files-found: ignore
# retention-days: 5

# - name: upload artifacts to S3
# run: src/ci/scripts/upload-artifacts.sh
# env:
# AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
# AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
# # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
# # builders *should* have the AWS credentials available. Still, explicitly
# # adding the condition is helpful as this way CI will not silently skip
# # deploying artifacts from a dist builder if the variables are misconfigured,
# # erroring about invalid credentials instead.
# if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'

# - name: upload job metrics to DataDog
# if: needs.calculate_matrix.outputs.run_type != 'pr'
# env:
# DATADOG_SITE: datadoghq.com
# DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
# DD_GITHUB_JOB_NAME: ${{ matrix.name }}
# run: |
# npm install -g @datadog/datadog-ci@^2.x.x
# python3 src/ci/scripts/upload-build-metrics.py build/cpu-usage.csv

# This job isused to tell bors the final status of the build, as there is no practical way to detect
# when a workflow is successful listening to webhooks only in our current bors implementation (homu).
outcome:
name: bors build finished
runs-on: ubuntu-latest
needs: [ calculate_matrix, job ]
# !cancelled() executes the job regardless of whether the previous jobs passed or failed
if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
steps:
- name: checkout the source code
uses: actions/checkout@v4
with:
fetch-depth: 2
# Calculate the exit status of the whole CI workflow.
# If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
# If a some dependent job has failed, this exits with 1.
- name: calculate the correct exit status
run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
# outcome:
# name: bors build finished
# runs-on: ubuntu-latest
# needs: [ calculate_matrix, job ]
# # !cancelled() executes the job regardless of whether the previous jobs passed or failed
# if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
# steps:
# - name: checkout the source code
# uses: actions/checkout@v4
# with:
# fetch-depth: 2
# # Calculate the exit status of the whole CI workflow.
# # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
# # If a some dependent job has failed, this exits with 1.
# - name: calculate the correct exit status
# run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
# Publish the toolstate if an auto build succeeds (just before push to master)
- name: publish toolstate
run: src/ci/publish_toolstate.sh
shell: bash
if: needs.calculate_matrix.outputs.run_type == 'auto'
env:
TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
TOOLSTATE_PUBLISH: 1
# - name: publish toolstate
# run: src/ci/publish_toolstate.sh
# shell: bash
# if: needs.calculate_matrix.outputs.run_type == 'auto'
# env:
# TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
# TOOLSTATE_PUBLISH: 1
1 change: 1 addition & 0 deletions src/ci/github-actions/calculate-job-matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ def get_custom_jobs(ctx: GitHubCtx) -> List[str]:


def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]:
return PRRunType()
if ctx.event_name == "pull_request":
return PRRunType()
elif ctx.event_name == "push":
Expand Down
111 changes: 96 additions & 15 deletions src/ci/github-actions/jobs.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,12 @@ runners:

envs:
env-x86_64-apple-tests: &env-x86_64-apple-tests
SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc
RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
SCRIPT: ./x.py --stage 2 build llvm
RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.download-rustc=false
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.12
MACOSX_STD_DEPLOYMENT_TARGET: 10.12
SELECT_XCODE: /Applications/Xcode_14.3.1.app
SELECT_XCODE: /Applications/Xcode_15.2.app
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
Expand Down Expand Up @@ -84,20 +84,101 @@ envs:
# These jobs automatically inherit envs.pr, to avoid repeating
# it in each job definition.
pr:
- image: mingw-check
# - image: mingw-check
# env:
# # We are adding (temporarily) a dummy commit on the compiler
# READ_ONLY_SRC: "0"
# <<: *job-linux-4c
# - image: mingw-check-tidy
# continue_on_error: true
# <<: *job-linux-4c
# - image: x86_64-gnu-llvm-18
# env:
# ENABLE_GCC_CODEGEN: "1"
# <<: *job-linux-16c
# - image: x86_64-gnu-tools
# <<: *job-linux-16c
- image: dist-x86_64-apple
env:
# We are adding (temporarily) a dummy commit on the compiler
READ_ONLY_SRC: "0"
<<: *job-linux-4c
- image: mingw-check-tidy
continue_on_error: true
<<: *job-linux-4c
- image: x86_64-gnu-llvm-18
SCRIPT: ./x.py build llvm --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1 --set rust.download-rustc=false
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.12
SELECT_XCODE: /Applications/Xcode_15.2.app
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
DIST_REQUIRE_ALL_TOOLS: 1
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-macos-xl

- image: dist-apple-various
env:
ENABLE_GCC_CODEGEN: "1"
<<: *job-linux-16c
- image: x86_64-gnu-tools
<<: *job-linux-16c
SCRIPT: ./x.py build llvm --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim,aarch64-apple-ios-macabi,x86_64-apple-ios-macabi
# Mac Catalyst cannot currently compile the sanitizer:
# https://github.com/rust-lang/rust/issues/129069
RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set target.aarch64-apple-ios-macabi.sanitizers=false --set target.x86_64-apple-ios-macabi.sanitizers=false --set rust.download-rustc=false
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.12
SELECT_XCODE: /Applications/Xcode_15.2.app
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
<<: *job-macos-xl

- image: x86_64-apple-1
env:
<<: *env-x86_64-apple-tests
<<: *job-macos-xl

- image: x86_64-apple-2
env:
SCRIPT: ./x.py --stage 2 build llvm
<<: *env-x86_64-apple-tests
<<: *job-macos-xl

# This target only needs to support 11.0 and up as nothing else supports the hardware
- image: dist-aarch64-apple
env:
SCRIPT: ./x.py build llvm --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin
RUST_CONFIGURE_ARGS: >-
--enable-full-tools
--enable-sanitizers
--enable-profiler
--set rust.jemalloc
--set llvm.ninja=false
--set rust.lto=thin
--set rust.download-rustc=false
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
SELECT_XCODE: /Applications/Xcode_15.4.app
USE_XCODE_CLANG: 1
MACOSX_DEPLOYMENT_TARGET: 11.0
MACOSX_STD_DEPLOYMENT_TARGET: 11.0
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
DIST_REQUIRE_ALL_TOOLS: 1
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-macos-m1

# This target only needs to support 11.0 and up as nothing else supports the hardware
- image: aarch64-apple
env:
SCRIPT: ./x.py --stage 2 build llvm --host=aarch64-apple-darwin --target=aarch64-apple-darwin
RUST_CONFIGURE_ARGS: >-
--enable-sanitizers
--enable-profiler
--set rust.jemalloc
--set rust.download-rustc=false
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
SELECT_XCODE: /Applications/Xcode_15.4.app
USE_XCODE_CLANG: 1
MACOSX_DEPLOYMENT_TARGET: 11.0
MACOSX_STD_DEPLOYMENT_TARGET: 11.0
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
<<: *job-macos-m1

# Jobs that run when you perform a try build (@bors try)
# These jobs automatically inherit envs.try, to avoid repeating
Expand Down

0 comments on commit 4bf068d

Please sign in to comment.