diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b4c30f3f1..d877f3f985 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ commands: - run: name: Install sccache command: | - export SCCACHE_VERSION='v0.4.2' + export SCCACHE_VERSION='v0.5.3' ls ~/.cargo/bin/sccache || curl -L https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz | tar -xOz sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl/sccache > ~/.cargo/bin/sccache && chmod +x ~/.cargo/bin/sccache # This configures Rust to use sccache. echo 'export "RUSTC_WRAPPER"="sccache"' >> $BASH_ENV @@ -304,7 +304,7 @@ jobs: default: false steps: - checkout - - run: + - run: name: Set git tag in bash_env command: | echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV @@ -347,40 +347,40 @@ jobs: type: boolean default: false steps: - - checkout - - run: - name: Set git tag in bash_env - command: | - echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV - - add_ssh_keys: - fingerprints: - - << parameters.ssh-fingerprint >> - - run: - name: Generate ssh config - command: ./.circleci/<< parameters.ssh-config-script >> - - run: - name: Deploy images - command: | - DOCKER_HOST=ssh://ec2-user@master.<< parameters.ssh-host >> USE_TLS=enable PROD=<< parameters.production >> DD_API_KEY=$DD_API_KEY \ - POSTGRES_PASSWORD=${<< parameters.postgres-password >>} \ - MONGO_INITDB_ROOT_PASSWORD=${<< parameters.mongodb-password >>} \ - TAG=$TAG \ - make deploy - - when: - condition: << parameters.production >> - steps: - - run: - name: Pull new deployer image on prod - command: | - ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle/deployer:$TAG" - - when: - condition: - not: << parameters.production >> - steps: - - run: - name: Pull new deployer image on dev - command: | - ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle-dev/deployer:$TAG" + - checkout + - run: + name: Set git tag in bash_env + command: | + echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV + - add_ssh_keys: + fingerprints: + - << parameters.ssh-fingerprint >> + - run: + name: Generate ssh config + command: ./.circleci/<< parameters.ssh-config-script >> + - run: + name: Deploy images + command: | + DOCKER_HOST=ssh://ec2-user@master.<< parameters.ssh-host >> USE_TLS=enable PROD=<< parameters.production >> DD_API_KEY=$DD_API_KEY \ + POSTGRES_PASSWORD=${<< parameters.postgres-password >>} \ + MONGO_INITDB_ROOT_PASSWORD=${<< parameters.mongodb-password >>} \ + TAG=$TAG \ + make deploy + - when: + condition: << parameters.production >> + steps: + - run: + name: Pull new deployer image on prod + command: | + ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle/deployer:$TAG" + - when: + condition: + not: << parameters.production >> + steps: + - run: + name: Pull new deployer image on dev + command: | + ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle-dev/deployer:$TAG" build-binaries-linux: machine: image: << parameters.image >> @@ -487,7 +487,7 @@ jobs: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - run: name: "Install Shuttle" - command: cargo install cargo-shuttle --path ./cargo-shuttle + command: cargo install cargo-shuttle --path ./cargo-shuttle - run: cargo shuttle --version - run: name: Login @@ -518,7 +518,7 @@ jobs: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - run: name: "Install Shuttle" - command: cargo install cargo-shuttle --path ./cargo-shuttle + command: cargo install cargo-shuttle --path ./cargo-shuttle - run: cargo shuttle --version - run: name: Login @@ -563,7 +563,7 @@ jobs: C:\rustup-init.exe -y --default-toolchain 1.70.0 --target x86_64-pc-windows-msvc - run: name: "Install Shuttle" - command: ..\.cargo\bin\cargo.exe install cargo-shuttle --path ./cargo-shuttle + command: ..\.cargo\bin\cargo.exe install cargo-shuttle --path ./cargo-shuttle - run: ..\.cargo\bin\cargo.exe shuttle --version - run: name: Login @@ -827,13 +827,13 @@ workflows: - publish-crates: matrix: parameters: - path: + path: [ "resources/aws-rds", "resources/shared-db", "resources/secrets", "resources/persist", - "resources/static-folder" + "resources/static-folder", ] name: publish-<< matrix.path >> requires: @@ -852,12 +852,12 @@ workflows: - publish-crates: matrix: parameters: - path: + path: [ - "services/shuttle-actix-web", - "services/shuttle-axum", - "services/shuttle-next", - "services/shuttle-poem", + "services/shuttle-actix-web", + "services/shuttle-axum", + "services/shuttle-next", + "services/shuttle-poem", "services/shuttle-poise", "services/shuttle-rocket", "services/shuttle-salvo", @@ -865,7 +865,7 @@ workflows: "services/shuttle-thruster", "services/shuttle-tide", "services/shuttle-tower", - "services/shuttle-warp" + "services/shuttle-warp", ] name: publish-<< matrix.path >> requires: diff --git a/Containerfile b/Containerfile index 325e360fa7..96197c5fda 100644 --- a/Containerfile +++ b/Containerfile @@ -1,47 +1,63 @@ -#syntax=docker/dockerfile-upstream:1.4.0-rc1 +#syntax=docker/dockerfile-upstream:1.4 + + +# Base image for builds and cache ARG RUSTUP_TOOLCHAIN FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-build -RUN apt-get update &&\ - apt-get install -y curl - -RUN cargo install cargo-chef +RUN cargo install cargo-chef --locked WORKDIR /build + +# Stores source cache FROM shuttle-build as cache +ARG CARGO_PROFILE WORKDIR /src COPY . . RUN find ${SRC_CRATES} \( -name "*.proto" -or -name "*.rs" -or -name "*.toml" -or -name "Cargo.lock" -or -name "README.md" -or -name "*.sql" \) -type f -exec install -D \{\} /build/\{\} \; -# This is used to carry over in the docker images any *.pem files from shuttle root directory, to be used for TLS testing, as described -# here in the admin README.md. -RUN [ "$CARGO_PROFILE" != "release" ] && find ${SRC_CRATES} -name "*.pem" -type f -exec install -D \{\} /build/\{\} \; +# This is used to carry over in the docker images any *.pem files from shuttle root directory, +# to be used for TLS testing, as described here in the admin README.md. +RUN [ "$CARGO_PROFILE" != "release" ] && \ + find ${SRC_CRATES} -name "*.pem" -type f -exec install -D \{\} /build/\{\} \; + +# Stores cargo chef recipe FROM shuttle-build AS planner COPY --from=cache /build . RUN cargo chef prepare --recipe-path recipe.json + +# Builds crate according to cargo chef recipe FROM shuttle-build AS builder -COPY --from=planner /build/recipe.json recipe.json ARG CARGO_PROFILE -RUN cargo chef cook $(if [ "$CARGO_PROFILE" = "release" ]; then echo --${CARGO_PROFILE}; fi) --recipe-path recipe.json -COPY --from=cache /build . ARG folder -# if CARGO_PROFILE is release, pass --release, else use default debug profile -RUN cargo build --bin shuttle-${folder} $(if [ "$CARGO_PROFILE" = "release" ]; then echo --${CARGO_PROFILE}; fi) - -ARG RUSTUP_TOOLCHAIN -FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common -RUN rustup component add rust-src +COPY --from=planner /build/recipe.json recipe.json +RUN cargo chef cook \ + # if CARGO_PROFILE is release, pass --release, else use default debug profile + $(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi) \ + --recipe-path recipe.json +COPY --from=cache /build . +RUN cargo build --bin shuttle-${folder} \ + $(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi) -COPY --from=cache /build/ /usr/src/shuttle/ -FROM shuttle-common +# The final image for this "shuttle-..." crate +ARG RUSTUP_TOOLCHAIN +FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-crate ARG folder ARG prepare_args +# used as env variable in prepare script ARG PROD -COPY ${folder}/prepare.sh /prepare.sh -RUN /prepare.sh "${prepare_args}" ARG CARGO_PROFILE -COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-${folder} /usr/local/bin/service ARG RUSTUP_TOOLCHAIN ENV RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN} + +COPY ${folder}/prepare.sh /prepare.sh +RUN /prepare.sh "${prepare_args}" + +COPY --from=cache /build /usr/src/shuttle/ + +# Any prepare steps that depend on the COPY from src cache +RUN /prepare.sh --after-src "${prepare_args}" + +COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-${folder} /usr/local/bin/service ENTRYPOINT ["/usr/local/bin/service"] diff --git a/README.md b/README.md index 5cd537a46a..248b444772 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ which will automatically install the correct target for your system. To install with `cargo-binstall`, run: ```sh +cargo install binstall cargo binstall cargo-shuttle ``` @@ -76,6 +77,8 @@ Although a bit slower, you can also install directly with cargo: cargo install cargo-shuttle ``` +*If installing binstall or cargo-shuttle fails, try adding `--locked` to the install command.* + After installing, log in with: ```sh diff --git a/auth/prepare.sh b/auth/prepare.sh index 6a52d3030c..f764443ec7 100755 --- a/auth/prepare.sh +++ b/auth/prepare.sh @@ -5,4 +5,9 @@ # service might need some extra preparation steps for its final image # ############################################################################### +# Stuff that depends on local source files +if [ "$1" = "--after-src" ]; then + exit 0 +fi + # Nothing to prepare in container image here diff --git a/deployer/prepare.sh b/deployer/prepare.sh index 21403d9e5c..4ee6728c5b 100755 --- a/deployer/prepare.sh +++ b/deployer/prepare.sh @@ -1,14 +1,27 @@ -#!/usr/bin/env sh +#!/bin/bash ############################################################################### # This file is used by our common Containerfile incase the container for this # # service might need some extra preparation steps for its final image # ############################################################################### + +# Stuff that depends on local source files +if [ "$1" = "--after-src" ]; then + + # Install the shuttle runtime + cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next + + exit 0 +fi + + # Patch crates to be on same versions mkdir -p $CARGO_HOME +touch $CARGO_HOME/config.toml if [[ $PROD != "true" ]]; then - echo '[patch.crates-io] + echo ' + [patch.crates-io] shuttle-codegen = { path = "/usr/src/shuttle/codegen" } shuttle-common = { path = "/usr/src/shuttle/common" } shuttle-proto = { path = "/usr/src/shuttle/proto" } @@ -21,8 +34,8 @@ if [[ $PROD != "true" ]]; then shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" } shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" } - shuttle-axum = { path = "/usr/src/shuttle/services/shuttle-axum" } shuttle-actix-web = { path = "/usr/src/shuttle/services/shuttle-actix-web" } + shuttle-axum = { path = "/usr/src/shuttle/services/shuttle-axum" } shuttle-next = { path = "/usr/src/shuttle/services/shuttle-next" } shuttle-poem = { path = "/usr/src/shuttle/services/shuttle-poem" } shuttle-poise = { path = "/usr/src/shuttle/services/shuttle-poise" } @@ -33,26 +46,14 @@ if [[ $PROD != "true" ]]; then shuttle-tide = { path = "/usr/src/shuttle/services/shuttle-tide" } shuttle-tower = { path = "/usr/src/shuttle/services/shuttle-tower" } shuttle-warp = { path = "/usr/src/shuttle/services/shuttle-warp" }' > $CARGO_HOME/config.toml -else - touch $CARGO_HOME/config.toml fi -# Install protoc since some users may need it -ARCH="linux-x86_64" && \ -VERSION="22.2" && \ -curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \ - unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \ - rm -f "protoc-$VERSION-$ARCH.zip" - # Add the wasm32-wasi target rustup target add wasm32-wasi -# Install the shuttle runtime -cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next - while getopts "p," o; do case $o in - "p") + "p") # if panamax is used, the '-p' parameter is passed # Make future crates requests to our own mirror echo ' [source.shuttle-crates-io-mirror] @@ -65,12 +66,14 @@ replace-with = "shuttle-crates-io-mirror"' >> $CARGO_HOME/config.toml esac done -# Prefetch crates.io index from our mirror -# TODO: restore when we know how to prefetch from our mirror -# cd /usr/src/shuttle/service -# cargo fetch - # Install common build tools for external crates # The image should already have these: https://github.com/docker-library/buildpack-deps/blob/65d69325ad741cea6dee20781c1faaab2e003d87/debian/buster/Dockerfile apt update -apt install -y llvm-dev libclang-dev clang cmake +apt install -y curl llvm-dev libclang-dev clang cmake + +# Install protoc since some users may need it +ARCH="linux-x86_64" && \ +VERSION="22.2" && \ +curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \ + unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \ + rm -f "protoc-$VERSION-$ARCH.zip" diff --git a/extras/postgres/Containerfile b/extras/postgres/Containerfile index 9e5047169b..11d4006687 100644 --- a/extras/postgres/Containerfile +++ b/extras/postgres/Containerfile @@ -2,8 +2,7 @@ ARG POSTGRES_TAG= FROM docker.io/postgres:${POSTGRES_TAG} -RUN apt-get update &&\ - apt-get install --yes curl python3 python3-aiohttp +RUN apt update && apt install -y curl python3 python3-aiohttp COPY watch /usr/sbin/watch COPY shuttle-entrypoint.sh /usr/local/bin/shuttle-entrypoint.sh diff --git a/gateway/prepare.sh b/gateway/prepare.sh index 6a52d3030c..f764443ec7 100755 --- a/gateway/prepare.sh +++ b/gateway/prepare.sh @@ -5,4 +5,9 @@ # service might need some extra preparation steps for its final image # ############################################################################### +# Stuff that depends on local source files +if [ "$1" = "--after-src" ]; then + exit 0 +fi + # Nothing to prepare in container image here diff --git a/provisioner/prepare.sh b/provisioner/prepare.sh index 6a52d3030c..f764443ec7 100755 --- a/provisioner/prepare.sh +++ b/provisioner/prepare.sh @@ -5,4 +5,9 @@ # service might need some extra preparation steps for its final image # ############################################################################### +# Stuff that depends on local source files +if [ "$1" = "--after-src" ]; then + exit 0 +fi + # Nothing to prepare in container image here diff --git a/service/src/builder.rs b/service/src/builder.rs index b9d1d4e4c0..35a5549ed6 100644 --- a/service/src/builder.rs +++ b/service/src/builder.rs @@ -144,12 +144,9 @@ pub async fn clean_crate(project_path: &Path, release_mode: bool) -> anyhow::Res let project_path = project_path.to_owned(); let manifest_path = project_path.join("Cargo.toml"); if !manifest_path.exists() { - return Err(anyhow!("failed to read the Shuttle project manifest")); - } - let mut profile = "dev"; - if release_mode { - profile = "release"; + bail!("failed to read the Shuttle project manifest"); } + let profile = if release_mode { "release" } else { "dev" }; let output = tokio::process::Command::new("cargo") .arg("clean") .arg("--manifest-path") @@ -221,6 +218,9 @@ async fn compile( tx: Sender, ) -> anyhow::Result> { let manifest_path = project_path.join("Cargo.toml"); + if !manifest_path.exists() { + bail!("failed to read the Shuttle project manifest"); + } let target_path = target_path.into(); let mut cargo = tokio::process::Command::new("cargo"); @@ -240,14 +240,13 @@ async fn compile( cargo.arg("--package").arg(package.name.clone()); } - let mut profile = "debug"; - - if release_mode { - profile = "release"; + let profile = if release_mode { cargo.arg("--profile").arg("release"); + "release" } else { cargo.arg("--profile").arg("dev"); - } + "debug" + }; if wasm { cargo.arg("--target").arg("wasm32-wasi"); @@ -283,8 +282,7 @@ async fn compile( target_path.clone(), "wasm32-wasi".into(), profile.into(), - #[allow(clippy::single_char_pattern)] - package.clone().name.replace("-", "_").into(), + package.name.replace('-', "_").into(), ] .iter() .collect();