diff --git a/.circleci/config.yml b/.circleci/config.yml index 659530ec0d..bcdda09f65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,6 @@ version: 2.1 # These "CircleCI Orbs" are reusable bits of configuration that can be shared # across projects. See https://circleci.com/orbs/ for more information. orbs: - gh: circleci/github-cli@2.6.2 slack: circleci/slack@5.1.1 secops: apollo/circleci-secops-orb@2.0.7 @@ -17,10 +16,13 @@ executors: environment: CARGO_BUILD_JOBS: 4 RUST_TEST_THREADS: 6 + MISE_ENV: ci amd_linux_helm: &amd_linux_helm_executor docker: - image: cimg/base:stable resource_class: small + environment: + MISE_ENV: ci amd_linux_test: &amd_linux_test_executor docker: - image: cimg/base:stable @@ -30,18 +32,21 @@ executors: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.17.0 resource_class: xlarge environment: + MISE_ENV: ci CARGO_BUILD_JOBS: 4 arm_linux_build: &arm_linux_build_executor machine: image: ubuntu-2004:2024.01.1 resource_class: arm.large environment: + MISE_ENV: ci CARGO_BUILD_JOBS: 8 arm_linux_test: &arm_linux_test_executor machine: image: ubuntu-2004:2024.01.1 resource_class: arm.xlarge environment: + MISE_ENV: ci CARGO_BUILD_JOBS: 8 macos_build: &macos_build_executor macos: @@ -50,6 +55,8 @@ executors: # We use the major.minor notation to bring in compatible patches. xcode: "15.4.0" resource_class: macos.m1.large.gen1 + environment: + MISE_ENV: ci,ci-mac macos_test: &macos_test_executor macos: # See https://circleci.com/docs/xcode-policy along with the support matrix @@ -57,31 +64,34 @@ executors: # We use the major.minor notation to bring in compatible patches. xcode: "15.4.0" resource_class: macos.m1.large.gen1 + environment: + MISE_ENV: ci,ci-mac windows_build: &windows_build_executor machine: image: "windows-server-2019-vs2019:2024.02.21" resource_class: windows.xlarge shell: bash.exe --login -eo pipefail + environment: + MISE_ENV: ci,windows windows_test: &windows_test_executor machine: image: "windows-server-2019-vs2019:2024.02.21" resource_class: windows.xlarge shell: bash.exe --login -eo pipefail + environment: + MISE_ENV: ci,windows # We don't use {{ arch }} because on windows it is unstable https://discuss.circleci.com/t/value-of-arch-unstable-on-windows/40079 parameters: toolchain_version: type: string - default: '{{ checksum ".circleci/config.yml" }}-v3-{{ checksum "~/.arch" }}-{{ checksum "rust-toolchain.toml" }}-{{ checksum "~/.daily_version" }}' + default: '{{ checksum ".circleci/config.yml" }}-v3-{{ checksum "~/.arch" }}-{{ checksum ".config/mise/config.toml" }}-{{ checksum "rust-toolchain.toml" }}-{{ checksum "~/.daily_version" }}' xtask_version: type: string - default: '{{ checksum ".circleci/config.yml" }}-v3-{{ checksum "~/.arch" }}-{{ checksum "rust-toolchain.toml" }}-{{ checksum "~/.xtask_version" }}' + default: '{{ checksum ".circleci/config.yml" }}-v3-{{ checksum "~/.arch" }}-{{ checksum ".config/mise/config.toml" }}-{{ checksum "rust-toolchain.toml" }}-{{ checksum "~/.xtask_version" }}' merge_version: type: string - default: '{{ checksum ".circleci/config.yml" }}-v3-{{ checksum "~/.arch" }}-{{ checksum "rust-toolchain.toml" }}-{{ checksum "~/.xtask_version" }}-{{ checksum "~/.merge_version" }}' - protoc_version: - type: string - default: "21.8" + default: '{{ checksum ".circleci/config.yml" }}-v3-{{ checksum "~/.arch" }}-{{ checksum ".config/mise/config.toml" }}-{{ checksum "rust-toolchain.toml" }}-{{ checksum "~/.xtask_version" }}-{{ checksum "~/.merge_version" }}' nightly: type: boolean default: false @@ -118,11 +128,8 @@ commands: - "<< pipeline.parameters.toolchain_version >>" - install_packages: platform: << parameters.platform >> - - install_protoc: - platform: << parameters.platform >> - - install_rust: + - install_mise: platform: << parameters.platform >> - - install_extra_tools - fetch_dependencies - save_cache: key: "<< pipeline.parameters.toolchain_version >>" @@ -149,6 +156,7 @@ commands: or: - equal: [ *amd_linux_build_executor, << parameters.platform >> ] - equal: [ *amd_linux_test_executor, << parameters.platform >> ] + - equal: [ *amd_linux_helm_executor, << parameters.platform >> ] steps: - run: name: Write arch @@ -186,11 +194,12 @@ commands: - equal: [ *windows_test_executor, << parameters.platform >> ] steps: - run: - name: Create bash aliases + name: Extend Bash profile for Windows command: | echo 'alias find=/bin/find' >> "$BASH_ENV" echo 'alias sort=/bin/sort' >> "$BASH_ENV" echo 'export EXECUTABLE_SUFFIX=".exe"' >> "$BASH_ENV" + echo 'export PATH="$HOME/AppData/Local/mise/shims:$HOME/scoop/apps/mise/current/bin:$HOME/scoop/shims:$PATH"' >> "$BASH_ENV" - run: name: Write arch command: | @@ -236,7 +245,8 @@ commands: sudo apt-get --download-only -o Dir::Cache="$HOME/.deb" -o Dir::Cache::archives="$HOME/.deb" install libssl-dev libdw-dev fi sudo dpkg -i $HOME/.deb/*.deb - install_protoc: + # Install mise for tool version management + install_mise: parameters: platform: type: executor @@ -244,72 +254,21 @@ commands: - when: condition: or: + - equal: [ *amd_linux_helm_executor, << parameters.platform >> ] - equal: [ *amd_linux_build_executor, << parameters.platform >> ] - equal: [ *amd_linux_test_executor, << parameters.platform >> ] - steps: - - run: - name: Install protoc - command: | - if [[ ! -f "$HOME/.local/bin/protoc" ]]; then - curl -L https://github.com/protocolbuffers/protobuf/releases/download/v<< pipeline.parameters.protoc_version >>/protoc-<< pipeline.parameters.protoc_version >>-linux-x86_64.zip --output protoc.zip - unzip protoc.zip -d $HOME/.local - fi - - when: - condition: - or: - equal: [ *arm_linux_build_executor, << parameters.platform >> ] - equal: [ *arm_linux_test_executor, << parameters.platform >> ] - steps: - - run: - name: Install protoc - command: | - if [[ ! -f "$HOME/.local/bin/protoc" ]]; then - curl -L https://github.com/protocolbuffers/protobuf/releases/download/v<< pipeline.parameters.protoc_version >>/protoc-<< pipeline.parameters.protoc_version >>-linux-aarch_64.zip --output protoc.zip - unzip protoc.zip -d $HOME/.local - fi - - when: - condition: - or: - equal: [ *macos_build_executor, << parameters.platform >> ] - equal: [ *macos_test_executor, << parameters.platform >> ] steps: - run: - name: Install protoc - command: | - if [[ ! -f "$HOME/.local/bin/protoc" ]]; then - curl -L https://github.com/protocolbuffers/protobuf/releases/download/v<< pipeline.parameters.protoc_version >>/protoc-<< pipeline.parameters.protoc_version >>-osx-universal_binary.zip --output protoc.zip - unzip protoc.zip -d $HOME/.local - fi - - when: - condition: - or: - - equal: [ *windows_build_executor, << parameters.platform >> ] - - equal: [ *windows_test_executor, << parameters.platform >> ] - steps: - - run: - name: Install protoc + name: Install mise command: | - if [[ ! -f "$HOME/.local/bin/protoc$EXECUTABLE_SUFFIX" ]]; then - curl -L https://github.com/protocolbuffers/protobuf/releases/download/v<< pipeline.parameters.protoc_version >>/protoc-<< pipeline.parameters.protoc_version >>-win64.zip --output protoc.zip - unzip protoc.zip -d $HOME/.local - fi - - install_rust: - parameters: - platform: - type: executor - steps: - - run: - name: Install Rust - command: | - if [[ ! -d "$HOME/.cargo" ]]; then - curl https://sh.rustup.rs -sSf -o rustup.sh - chmod 755 ./rustup.sh - ./rustup.sh -y --profile minimal --component clippy --component rustfmt --default-toolchain none - $HOME/.cargo/bin/rustc -V - fi - echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" - + curl https://mise.jdx.dev/install.sh | sh + mise activate bash >> "$BASH_ENV" + mise trust + mise install --yes - when: condition: or: @@ -317,49 +276,22 @@ commands: - equal: [ *windows_test_executor, << parameters.platform >> ] steps: - run: - name: Special case for Windows because of ssh-agent - command: | - printf "[net]\ngit-fetch-with-cli = true" >> ~/.cargo/Cargo.toml - - when: - condition: - or: - - equal: [ *macos_build_executor, << parameters.platform >> ] - steps: - - run: - name: Special case for OSX x86_64 builds + name: Install scoop + shell: powershell.exe -ExecutionPolicy Bypass command: | - rustup target add x86_64-apple-darwin - - - when: - condition: - equal: [ *arm_linux_test_executor, << parameters.platform >> ] - steps: + iex "& {$(irm get.scoop.sh)} -RunAsAdmin" - run: - name: Install nightly Rust to build the fuzzers + name: Install mise command: | - rustup install nightly - - install_extra_tools: - steps: - - run: - name: Install cargo deny, about, edit - command: | - if [[ ! -f "$HOME/.cargo/bin/cargo-deny$EXECUTABLE_SUFFIX" ]]; then - cargo install --locked --version 0.14.21 cargo-deny - cargo install --locked --version 0.12.2 cargo-edit - cargo install --locked --version 0.12.0 cargo-fuzz - cargo install --locked --version 0.6.6 cargo-about - fi - - if [[ ! -f "$HOME/.cargo/bin/cargo-nextest$EXECUTABLE_SUFFIX" ]]; then - cargo install --locked --version 0.9.70 cargo-nextest - fi - + scoop install mise + mise trust + mise install --yes fetch_dependencies: steps: - run: name: Fetch dependencies command: cargo fetch --locked + install_xtask: steps: - restore_cache: @@ -382,7 +314,7 @@ commands: - restore_cache: keys: - "<< pipeline.parameters.merge_version >>-lint" - - run: xtask lint + - run: $HOME/.cargo/bin/xtask$EXECUTABLE_SUFFIX lint - when: condition: equal: [ "dev", "<< pipeline.git.branch >>" ] @@ -400,13 +332,6 @@ commands: - run: name: Validate helm manifests command: | - # Install Helm - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - # Install kubeconform - KUBECONFORM_INSTALL=$(mktemp -d) - curl -L https://github.com/yannh/kubeconform/releases/latest/download/kubeconform-linux-amd64.tar.gz | tar xz -C "${KUBECONFORM_INSTALL}" - # Create list of kube versions CURRENT_KUBE_VERSIONS=$(curl -s -L https://raw.githubusercontent.com/kubernetes/website/main/data/releases/schedule.yaml \ | yq -o json '.' \ @@ -441,7 +366,7 @@ commands: helm template --kube-version "${kube_version}" router helm/chart/router --set autoscaling.enabled=true > "${TEMPLATE_DIR}/router-${kube_version}.yaml" # Execute kubeconform on our templated charts to ensure they are good - "${KUBECONFORM_INSTALL}/kubeconform" \ + kubeconform \ --kubernetes-version "${kube_version}" \ --strict \ --schema-location default \ @@ -457,7 +382,7 @@ commands: - "<< pipeline.parameters.merge_version >>-compliance" # cargo-deny fetches a rustsec advisory DB, which has to happen on github.com over https - run: git config --global --unset-all url.ssh://git@github.com.insteadof - - run: xtask check-compliance + - run: $HOME/.cargo/bin/xtask$EXECUTABLE_SUFFIX check-compliance - when: condition: equal: [ "dev", "<< pipeline.git.branch >>" ] @@ -480,7 +405,11 @@ commands: environment: # Use the settings from the "ci" profile in nextest configuration. NEXTEST_PROFILE: ci - command: xtask test --workspace --locked --features ci,hyper_header_limits + # Temporary disable lib backtrace since it crashing on MacOS + # TODO: remove this workaround once we update to Xcode >= 15.1.0 + # See: https://github.com/apollographql/router/pull/5462 + RUST_LIB_BACKTRACE: 0 + command: $HOME/.cargo/bin/xtask$EXECUTABLE_SUFFIX test --workspace --locked --features ci,hyper_header_limits - run: name: Delete large files from cache command: | @@ -528,6 +457,8 @@ jobs: equal: [ *amd_linux_helm_executor, << parameters.platform >> ] steps: - checkout + - setup_environment: + platform: << parameters.platform >> - xtask_check_helm check_compliance: @@ -722,25 +653,6 @@ jobs: --keychain-password ${MACOS_KEYCHAIN_PASSWORD} --notarization-password ${MACOS_NOTARIZATION_PASSWORD} --output artifacts/ - - when: - condition: - and: - - equal: [ *amd_linux_build_executor, << parameters.platform >> ] - - equal: [ true, << parameters.nightly >> ] - steps: - - run: - name: Helm install - command: | - # Install Helm - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - run: - name: helm-docs install - command: | - # install golang (to ${HOME}/go) - curl -OLs https://go.dev/dl/go1.21.3.linux-amd64.tar.gz - tar -C "${HOME}" -xf go1.21.3.linux-amd64.tar.gz - # install helm-docs - PATH="${HOME}/go/bin" GOPATH="${HOME}/.local" GO111MODULE=on go install github.com/norwoodj/helm-docs/cmd/helm-docs@latest - when: condition: or: @@ -918,6 +830,8 @@ jobs: equal: [ "https://github.com/apollographql/router", << pipeline.project.git_url >> ] steps: - checkout + - setup_environment: + platform: amd_linux_build - setup_remote_docker: # CircleCI Image Policy # https://circleci.com/docs/remote-docker-images-support-policy/ @@ -925,7 +839,6 @@ jobs: docker_layer_caching: true - attach_workspace: at: artifacts - - gh/setup - run: command: > cd artifacts && sha256sum *.tar.gz > sha256sums.txt @@ -970,8 +883,6 @@ jobs: - run: name: Helm build command: | - # Install Helm - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash # Package up the helm chart helm package helm/chart/router # Make sure we have the newest chart diff --git a/.config/mise/config.ci-mac.toml b/.config/mise/config.ci-mac.toml new file mode 100644 index 0000000000..d4c01f837b --- /dev/null +++ b/.config/mise/config.ci-mac.toml @@ -0,0 +1,3 @@ +[tools] +# renovate-automation: rustc version +rust = { version = "1.85.1", targets = "x86_64-apple-darwin,aarch64-apple-darwin" } diff --git a/.config/mise/config.ci.toml b/.config/mise/config.ci.toml new file mode 100644 index 0000000000..c0b0b248fe --- /dev/null +++ b/.config/mise/config.ci.toml @@ -0,0 +1,3 @@ +[tools] +"cargo:cargo-fuzz" = "0.12.0" +kubeconform = "0.6.7" diff --git a/.config/mise/config.toml b/.config/mise/config.toml index 7ae59e7137..607c80ddcb 100644 --- a/.config/mise/config.toml +++ b/.config/mise/config.toml @@ -1,5 +1,6 @@ [tools] rust = "1.85.0" +"aqua:cargo-bins/cargo-binstall" = "1.12.3" "cargo:cargo-nextest" = "0.9.70" "cargo:cargo-deny" = "0.14.21" "cargo:cargo-edit" = "0.12.2" @@ -7,9 +8,9 @@ rust = "1.85.0" "cargo:cargo-insta" = "1.38.0" "cargo:htmlq" = "0.4.0" "cargo:cargo-watch" = "8.5.3" -"cargo:graphql_client_cli" = "0.14.0" -"cargo:cargo-fuzz" = "0.12.0" +"cargo:typos-cli" = "1.31.1" protoc = "21.8" +gh = "2.72.0" helm = "3.17.3" helm-docs = "1.14.2" yq = "4.45.1" diff --git a/.config/mise/config.windows.toml b/.config/mise/config.windows.toml new file mode 100644 index 0000000000..750983275c --- /dev/null +++ b/.config/mise/config.windows.toml @@ -0,0 +1,2 @@ +[tools] +cmake = "3.31.1"