Skip to content

Commit

Permalink
ci fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonaro00
jonaro00 committed Aug 21, 2023
1 parent 3462118 commit 748acaa
Showing 1 changed file with 89 additions and 67 deletions.
156 changes: 89 additions & 67 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,41 +22,46 @@ executors:
commands:
restore-cargo-cache:
steps:
# Restore cargo cache before installing anything with the cargo command (ie cargo install ...)
- restore_cache:
name: Restore cargo cache
keys:
- cargo-{{ checksum "Cargo.lock" }}
name: Restore cargo registry cache
key: cargo-{{ checksum "Cargo.lock" }}
# Don't use fallback key to prevent registry cache from growing indefinitely.
- run:
name: Install sccache
# 500 MB is the recommended soft cap for a cache on circleci.
# Large integration tests override this default.
command: |
export SCCACHE_VERSION='v0.5.3'
ls ~/.cargo/bin/sccache || curl -L https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz | tar -xOz sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl/sccache > ~/.cargo/bin/sccache && chmod +x ~/.cargo/bin/sccache
# This configures Rust to use sccache.
echo 'export "RUSTC_WRAPPER"="sccache"' >> $BASH_ENV
sccache --version
SCCACHE_VERSION='v0.5.4'
ls ~/.cargo/bin/sccache \
|| curl -L https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz \
| tar -xOz sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl/sccache \
> ~/.cargo/bin/sccache \
&& chmod +x ~/.cargo/bin/sccache
echo 'export RUSTC_WRAPPER=~/.cargo/bin/sccache' >> $BASH_ENV
echo 'export SCCACHE_CACHE_SIZE=500M' >> $BASH_ENV
- restore_cache:
name: Restore sccache cache
key: sccache-cache-{{ .Environment.CIRCLE_JOB }}
keys: # This can use fallback due to size limit.
- sccache-cache-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-
- sccache-cache-{{ .Environment.CIRCLE_JOB }}-
save-cargo-cache:
steps:
- run:
name: Sccache stats
command: sccache --show-stats
- save_cache:
name: Save sccache cache
# We use {{ epoch }} to always upload a fresh cache:
# Of course, restore_cache will not find this exact key,
# but it will fall back to the closest key (aka the most recent).
# See https://discuss.circleci.com/t/add-mechanism-to-update-existing-cache-key/9014/13
key: sccache-cache-{{ .Environment.CIRCLE_JOB }}-{{ epoch }}
# We use {{ .Branch }}-{{ .Revision }} to upload a fresh cache for each commit on a branch.
# If a new commit is built, it will fall back on the most recent cache from the same branch.
key: sccache-cache-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ .Revision }}
paths:
- "~/.cache/sccache"
- save_cache:
name: Save cargo cache
key: cargo-{{ checksum "Cargo.lock" }}-{{ epoch }}
key: cargo-{{ checksum "Cargo.lock" }}
paths:
- ~/.cargo
- ~/.cargo/registry
- ~/.cargo/git
restore-buildx-cache:
steps:
- docker-buildx/install:
Expand All @@ -66,11 +71,9 @@ commands:
name: Restore buildx cache
keys:
# Try lock cache first
- docker-buildx-{{ checksum "./Cargo.lock" }}
- docker-buildx-{{ checksum "Cargo.lock" }}
# Try branch cache next
- docker-buildx-{{ .Branch }}
# Fallback to main cache
- docker-buildx-main
- run:
name: Configure buildx cache
command: export BUILDX_CACHE=/tmp/cache/buildx
Expand All @@ -80,7 +83,7 @@ commands:
name: Save buildx cache
paths:
- "/tmp/cache/buildx"
key: docker-buildx-{{ checksum "./Cargo.lock" }}-{{ epoch }}
key: docker-buildx-{{ checksum "Cargo.lock" }}-{{ epoch }}
when: always
apply-patches:
steps:
Expand Down Expand Up @@ -121,8 +124,9 @@ commands:
- run:
name: Install Rust
command: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.70.0 --target add wasm32-wasi
sudo apt update && sudo apt install -y libssl1.1
which cargo || curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.70.0
# sudo apt update && sudo apt install -y libssl1.1
- run: rustup target add wasm32-wasi
install-protoc:
steps:
- run:
Expand All @@ -132,7 +136,7 @@ commands:
VERSION: "22.2"
command: |
curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \
unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \
sudo unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \
rm -f "protoc-$VERSION-$ARCH.zip"
make-artifact:
parameters:
Expand Down Expand Up @@ -170,24 +174,6 @@ commands:
paths:
- << parameters.target >>/*
- << parameters.target >>.env
do-integration:
steps:
- install-rust
- checkout
- run: git submodule sync
- run: git submodule update --init
- restore-cargo-cache
- apply-patches
- run:
name: Run unit tests
command: cargo test --package << parameters.crate >> --all-features --lib -- --nocapture
- run:
name: Run integration tests
# Only run integration tests if there are any
command: |
set +o pipefail
(cargo test --package << parameters.crate >> --all-features --test '*' -- --list 2>&1 | grep -q "no test target matches pattern") && echo "nothing to test" || cargo test --package << parameters.crate >> --all-features --test '*' -- --nocapture
- save-cargo-cache

jobs:
workspace:
Expand All @@ -198,22 +184,19 @@ jobs:
- restore-cargo-cache
# Check this to make sure we do not include patched dependencies in
# the Cargo.lock.
- run:
command: |
[ -z $(grep "\[\[patch.unused\]\]" Cargo.lock) ]
- run: '[ -z $(grep "\[\[patch.unused\]\]" Cargo.lock) ]'
- run: cargo fmt --all --check
# TODO: this is incompatible with workspace inheritance, uncomment when
# https://github.com/DevinR528/cargo-sort/pull/29 is merged
# - run: cargo install cargo-sort
# - run: # install binstall with script
# - run: cargo binstall cargo-sort
# - run: cargo sort --check --workspace
- run: |
cargo clippy --tests \
--all-targets \
--all-features \
--no-deps -- \
--D warnings \
-A clippy::let-unit-value \
-A clippy::format-push-string
--D warnings
- save-cargo-cache
test-standalone:
parameters:
Expand All @@ -232,7 +215,8 @@ jobs:
- run: cargo fmt --all --check --manifest-path << parameters.path >>/Cargo.toml
# TODO: this is incompatible with workspace inheritance, uncomment when
# https://github.com/DevinR528/cargo-sort/pull/29 is merged
# - run: cargo install cargo-sort
# - run: # install binstall with script
# - run: cargo binstall cargo-sort
# - run: cargo sort --check << parameters.path >>
- run: |
cargo clippy --tests \
Expand All @@ -241,16 +225,20 @@ jobs:
--manifest-path << parameters.path >>/Cargo.toml \
--no-deps -- \
--D warnings
- run: cargo test << parameters.features >> --manifest-path << parameters.path >>/Cargo.toml -- --nocapture
- run: |
cargo test << parameters.features >> \
--manifest-path << parameters.path >>/Cargo.toml \
-- \
--nocapture
- save-cargo-cache
test-workspace-member:
parameters:
crate:
description: "Crate in workspace to test"
type: string
executor: docker-rust
resource_class: medium+
steps:
- run: rustup target add wasm32-wasi
- checkout
- restore-cargo-cache
- run: cargo test --package << parameters.crate >> --all-features -- --nocapture
Expand All @@ -261,9 +249,26 @@ jobs:
description: "Crate in workspace to test"
type: string
executor: docker-rust
resource_class: medium+
resource_class: large
steps:
- do-integration
- install-rust
- install-protoc
- checkout
- run: git submodule update --init
- restore-cargo-cache
- apply-patches
- run:
name: Run unit tests
command: cargo test --package << parameters.crate >> --all-features --lib -- --nocapture
- run:
name: Run integration tests
# Only run integration tests if there are any
command: |
set +o pipefail
(cargo test --package << parameters.crate >> --all-features --test '*' -- --list 2>&1 | grep -q "no test target matches pattern") \
&& echo "nothing to test" \
|| cargo test --package << parameters.crate >> --all-features --test '*' -- --nocapture
- save-cargo-cache
test-workspace-member-integration-machine:
parameters:
crate:
Expand All @@ -272,13 +277,32 @@ jobs:
# Using a machine image since tests will start a docker container
executor: machine-ubuntu
steps:
- do-integration
- install-rust
- install-protoc
- checkout
- run: git submodule update --init
- restore-cargo-cache
- apply-patches
- run:
name: Run unit tests
command: |
SCCACHE_CACHE_SIZE=2G
cargo test --package << parameters.crate >> --all-features --lib -- --nocapture
- run:
name: Run integration tests
# Only run integration tests if there are any
command: |
SCCACHE_CACHE_SIZE=2G
set +o pipefail
(cargo test --package << parameters.crate >> --all-features --test '*' -- --list 2>&1 | grep -q "no test target matches pattern") \
&& echo "nothing to test" \
|| cargo test --package << parameters.crate >> --all-features --test '*' -- --nocapture
- save-cargo-cache
e2e-test:
executor: machine-ubuntu
steps:
- install-rust
- checkout
- run: git submodule sync
- run: git submodule update --init
- restore-buildx-cache
- run:
Expand All @@ -298,11 +322,13 @@ jobs:
SHUTTLE_TESTS_NETWORK=shuttle-dev_user-net SHUTTLE_TESTS_RUNTIME_IMAGE=public.ecr.aws/shuttle-dev/deployer:latest cargo test --package shuttle-gateway --all-features --lib -- --nocapture
# runs tests in e2e crate
BUILDX_CACHE=/tmp/cache/buildx make test
- save_cache:
name: Save buildx cache
paths:
- "/tmp/cache/buildx"
key: docker-buildx-{{ .Branch }}
key: docker-buildx-{{ .Branch }}-{{ .Revision }}
when: always
build-and-push:
executor: machine-ubuntu
Expand Down Expand Up @@ -497,8 +523,8 @@ jobs:
name: "Set tag in environment"
command: |
for file in artifacts/*.env; do
cat artifacts/${file##*/} >> "$BASH_ENV";
rm artifacts/${file##*/};
cat artifacts/${file##*/} >> $BASH_ENV
rm artifacts/${file##*/}
done
- run:
name: "Publish Release on GitHub"
Expand Down Expand Up @@ -531,7 +557,6 @@ jobs:
command: ./.circleci/qa.sh linux
environment:
SHUTTLE_API: https://api.unstable.shuttle.rs
- run: git submodule sync
- run: git submodule update --init
- run:
name: Test WASM
Expand Down Expand Up @@ -562,7 +587,6 @@ jobs:
command: ./.circleci/qa.sh mac
environment:
SHUTTLE_API: https://api.unstable.shuttle.rs
- run: git submodule sync
- run: git submodule update --init
- run:
name: Test WASM
Expand Down Expand Up @@ -607,7 +631,6 @@ jobs:
command: ./.circleci/qa.ps1
environment:
SHUTTLE_API: https://api.unstable.shuttle.rs
- run: git submodule sync
- run: git submodule update --init
# - run:
# name: Test WASM
Expand Down Expand Up @@ -658,8 +681,7 @@ workflows:
- services/shuttle-tower
- services/shuttle-warp
- test-standalone:
# shuttle-shared-db has mutually exclusive features
# so we run checks for each feature separately
# Has mutually exclusive features, so we run checks for each feature separately
name: "resources/shared-db: << matrix.features >>"
matrix:
alias: test-standalone-shared-db
Expand All @@ -672,7 +694,7 @@ workflows:
- "-F postgres-rustls"
############ This crate has no tests, therefore excluded.
# - test-standalone:
# # Has mutually exclusive features, so we run checks for each feature separately
# # Has mutually exclusive features, so we run checks for each feature separately
# name: "resources/aws-rds: << matrix.features >>"
# matrix:
# alias: test-standalone-aws-rds
Expand Down Expand Up @@ -703,7 +725,6 @@ workflows:
- shuttle-codegen
- shuttle-deployer
- shuttle-proto
- shuttle-provisioner
- shuttle-resource-recorder
- shuttle-runtime
- shuttle-service
Expand All @@ -712,8 +733,9 @@ workflows:
matrix:
parameters:
crate:
# needs to spawn docker containers, therefore in a machine
# need to spawn docker containers, therefore in a machine
- cargo-shuttle
- shuttle-provisioner
- e2e-test:
requires:
- test-standalone
Expand Down

0 comments on commit 748acaa

Please sign in to comment.