fix: dockerfile and ci improvements (#989)

* cleanup in dockerfiles and ci * minor builder fixes * add sccache to deployer + cleanup * fmt * docs: install with --locked * Add comments, move arg statements * Leave out the sccache * clarify * Run prepare script before loading src cache * Delay src-dependent preparation to after cache * explicit docker.io, move curl installation --------- Co-authored-by: oddgrd <29732646+oddgrd@users.noreply.github.com>
shuttle-hq · Jun 26, 2023 · 947d6a7 · 947d6a7
1 parent 9907349
commit 947d6a7
Showing 9 changed files with 142 additions and 108 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -29,7 +29,7 @@ commands:
       - run:
           name: Install sccache
           command: |
-            export SCCACHE_VERSION='v0.4.2'
+            export SCCACHE_VERSION='v0.5.3'
             ls ~/.cargo/bin/sccache || curl -L https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz | tar -xOz sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl/sccache > ~/.cargo/bin/sccache && chmod +x ~/.cargo/bin/sccache
             # This configures Rust to use sccache.
             echo 'export "RUSTC_WRAPPER"="sccache"' >> $BASH_ENV
@@ -299,7 +299,7 @@ jobs:
         default: false
     steps:
       - checkout
-      - run: 
+      - run:
           name: Set git tag in bash_env
           command: |
             echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV
@@ -342,40 +342,40 @@ jobs:
         type: boolean
         default: false
     steps:
-    - checkout
-    - run:
-        name: Set git tag in bash_env
-        command: |
-          echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV
-    - add_ssh_keys:
-        fingerprints:
-          - << parameters.ssh-fingerprint >>
-    - run:
-        name: Generate ssh config
-        command: ./.circleci/<< parameters.ssh-config-script >>
-    - run:
-        name: Deploy images
-        command: |
-          DOCKER_HOST=ssh://ec2-user@master.<< parameters.ssh-host >> USE_TLS=enable PROD=<< parameters.production >> DD_API_KEY=$DD_API_KEY \
-          POSTGRES_PASSWORD=${<< parameters.postgres-password >>} \
-          MONGO_INITDB_ROOT_PASSWORD=${<< parameters.mongodb-password >>} \
-          TAG=$TAG \
-          make deploy
-    - when:
-        condition: << parameters.production >>
-        steps:
-          - run:
-              name: Pull new deployer image on prod
-              command: |
-                ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle/deployer:$TAG"
-    - when:
-        condition:
-          not: << parameters.production >>
-        steps:
-          - run:
-              name: Pull new deployer image on dev
-              command: |
-                ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle-dev/deployer:$TAG"
+      - checkout
+      - run:
+          name: Set git tag in bash_env
+          command: |
+            echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV
+      - add_ssh_keys:
+          fingerprints:
+            - << parameters.ssh-fingerprint >>
+      - run:
+          name: Generate ssh config
+          command: ./.circleci/<< parameters.ssh-config-script >>
+      - run:
+          name: Deploy images
+          command: |
+            DOCKER_HOST=ssh://ec2-user@master.<< parameters.ssh-host >> USE_TLS=enable PROD=<< parameters.production >> DD_API_KEY=$DD_API_KEY \
+            POSTGRES_PASSWORD=${<< parameters.postgres-password >>} \
+            MONGO_INITDB_ROOT_PASSWORD=${<< parameters.mongodb-password >>} \
+            TAG=$TAG \
+            make deploy
+      - when:
+          condition: << parameters.production >>
+          steps:
+            - run:
+                name: Pull new deployer image on prod
+                command: |
+                  ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle/deployer:$TAG"
+      - when:
+          condition:
+            not: << parameters.production >>
+          steps:
+            - run:
+                name: Pull new deployer image on dev
+                command: |
+                  ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle-dev/deployer:$TAG"
   build-binaries-linux:
     machine:
       image: << parameters.image >>
@@ -482,7 +482,7 @@ jobs:
             curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - run:
           name: "Install Shuttle"
-          command:  cargo install cargo-shuttle --path ./cargo-shuttle
+          command: cargo install cargo-shuttle --path ./cargo-shuttle
       - run: cargo shuttle --version
       - run:
           name: Login
@@ -513,7 +513,7 @@ jobs:
             curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - run:
           name: "Install Shuttle"
-          command:  cargo install cargo-shuttle --path ./cargo-shuttle
+          command: cargo install cargo-shuttle --path ./cargo-shuttle
       - run: cargo shuttle --version
       - run:
           name: Login
@@ -558,7 +558,7 @@ jobs:
             C:\rustup-init.exe -y --default-toolchain 1.70.0 --target x86_64-pc-windows-msvc
       - run:
           name: "Install Shuttle"
-          command:  ..\.cargo\bin\cargo.exe install cargo-shuttle --path ./cargo-shuttle
+          command: ..\.cargo\bin\cargo.exe install cargo-shuttle --path ./cargo-shuttle
       - run: ..\.cargo\bin\cargo.exe shuttle --version
       - run:
           name: Login
@@ -618,8 +618,8 @@ workflows:
                 - services/shuttle-tower
                 - services/shuttle-warp
       - check-standalone:
-        # shuttle-shared-db has mutually exclusive features
-        # so we run checks for each feature separately
+          # shuttle-shared-db has mutually exclusive features
+          # so we run checks for each feature separately
           name: "resources/shared-db: << matrix.features >>"
           matrix:
             alias: check-standalone-shared-db
@@ -645,7 +645,7 @@ workflows:
                   "shuttle-proto",
                   "shuttle-provisioner",
                   "shuttle-runtime",
-                  "shuttle-service"
+                  "shuttle-service",
                 ]
       - e2e-test:
           requires:
@@ -808,13 +808,13 @@ workflows:
       - publish-crates:
           matrix:
             parameters:
-              path: 
+              path:
                 [
                   "resources/aws-rds",
                   "resources/shared-db",
                   "resources/secrets",
                   "resources/persist",
-                  "resources/static-folder"
+                  "resources/static-folder",
                 ]
           name: publish-<< matrix.path >>
           requires:
@@ -833,20 +833,20 @@ workflows:
       - publish-crates:
           matrix:
             parameters:
-              path: 
+              path:
                 [
-                  "services/shuttle-actix-web", 
-                  "services/shuttle-axum", 
-                  "services/shuttle-next", 
-                  "services/shuttle-poem", 
+                  "services/shuttle-actix-web",
+                  "services/shuttle-axum",
+                  "services/shuttle-next",
+                  "services/shuttle-poem",
                   "services/shuttle-poise",
                   "services/shuttle-rocket",
                   "services/shuttle-salvo",
                   "services/shuttle-serenity",
                   "services/shuttle-thruster",
                   "services/shuttle-tide",
                   "services/shuttle-tower",
-                  "services/shuttle-warp"
+                  "services/shuttle-warp",
                 ]
           name: publish-<< matrix.path >>
           requires:

diff --git a/Containerfile b/Containerfile
@@ -1,47 +1,63 @@
-#syntax=docker/dockerfile-upstream:1.4.0-rc1
+#syntax=docker/dockerfile-upstream:1.4
+
+
+# Base image for builds and cache
 ARG RUSTUP_TOOLCHAIN
 FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-build
-RUN apt-get update &&\
-    apt-get install -y curl
-
-RUN cargo install cargo-chef
+RUN cargo install cargo-chef --locked
 WORKDIR /build
 
+
+# Stores source cache
 FROM shuttle-build as cache
+ARG CARGO_PROFILE
 WORKDIR /src
 COPY . .
 RUN find ${SRC_CRATES} \( -name "*.proto" -or -name "*.rs" -or -name "*.toml" -or -name "Cargo.lock" -or -name "README.md" -or -name "*.sql" \) -type f -exec install -D \{\} /build/\{\} \;
-# This is used to carry over in the docker images any *.pem files from shuttle root directory, to be used for TLS testing, as described
-# here in the admin README.md.
-RUN [ "$CARGO_PROFILE" != "release" ] && find ${SRC_CRATES} -name "*.pem" -type f -exec install -D \{\} /build/\{\} \;
+# This is used to carry over in the docker images any *.pem files from shuttle root directory,
+# to be used for TLS testing, as described here in the admin README.md.
+RUN [ "$CARGO_PROFILE" != "release" ] && \
+    find ${SRC_CRATES} -name "*.pem" -type f -exec install -D \{\} /build/\{\} \;
+
 
+# Stores cargo chef recipe
 FROM shuttle-build AS planner
 COPY --from=cache /build .
 RUN cargo chef prepare --recipe-path recipe.json
 
+
+# Builds crate according to cargo chef recipe
 FROM shuttle-build AS builder
-COPY --from=planner /build/recipe.json recipe.json
 ARG CARGO_PROFILE
-RUN cargo chef cook $(if [ "$CARGO_PROFILE" = "release" ]; then echo --${CARGO_PROFILE}; fi) --recipe-path recipe.json
-COPY --from=cache /build .
 ARG folder
-# if CARGO_PROFILE is release, pass --release, else use default debug profile
-RUN cargo build --bin shuttle-${folder} $(if [ "$CARGO_PROFILE" = "release" ]; then echo --${CARGO_PROFILE}; fi)
-
-ARG RUSTUP_TOOLCHAIN
-FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common
-RUN rustup component add rust-src
+COPY --from=planner /build/recipe.json recipe.json
+RUN cargo chef cook \
+    # if CARGO_PROFILE is release, pass --release, else use default debug profile
+    $(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi) \
+    --recipe-path recipe.json
+COPY --from=cache /build .
+RUN cargo build --bin shuttle-${folder} \
+    $(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi)
 
-COPY --from=cache /build/ /usr/src/shuttle/
 
-FROM shuttle-common
+# The final image for this "shuttle-..." crate
+ARG RUSTUP_TOOLCHAIN
+FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-crate
 ARG folder
 ARG prepare_args
+# used as env variable in prepare script
 ARG PROD
-COPY ${folder}/prepare.sh /prepare.sh
-RUN /prepare.sh "${prepare_args}"
 ARG CARGO_PROFILE
-COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-${folder} /usr/local/bin/service
 ARG RUSTUP_TOOLCHAIN
 ENV RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN}
+
+COPY ${folder}/prepare.sh /prepare.sh
+RUN /prepare.sh "${prepare_args}"
+
+COPY --from=cache /build /usr/src/shuttle/
+
+# Any prepare steps that depend on the COPY from src cache
+RUN /prepare.sh --after-src "${prepare_args}"
+
+COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-${folder} /usr/local/bin/service
 ENTRYPOINT ["/usr/local/bin/service"]
diff --git a/README.md b/README.md
@@ -67,6 +67,7 @@ which will automatically install the correct target for your system.
 To install with `cargo-binstall`, run:
 
 ```sh
+cargo install binstall
 cargo binstall cargo-shuttle
 ```
 
@@ -76,6 +77,8 @@ Although a bit slower, you can also install directly with cargo:
 cargo install cargo-shuttle
 ```
 
+*If installing binstall or cargo-shuttle fails, try adding `--locked` to the install command.*
+
 After installing, log in with:
 
 ```sh

diff --git a/auth/prepare.sh b/auth/prepare.sh
@@ -5,4 +5,9 @@
 # service might need some extra preparation steps for its final image         #
 ###############################################################################
 
+# Stuff that depends on local source files
+if [ "$1" = "--after-src" ]; then
+    exit 0
+fi
+
 # Nothing to prepare in container image here
diff --git a/deployer/prepare.sh b/deployer/prepare.sh
@@ -1,14 +1,27 @@
-#!/usr/bin/env sh
+#!/bin/bash
 
 ###############################################################################
 # This file is used by our common Containerfile incase the container for this #
 # service might need some extra preparation steps for its final image         #
 ###############################################################################
 
+
+# Stuff that depends on local source files
+if [ "$1" = "--after-src" ]; then
+
+    # Install the shuttle runtime
+    cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next
+
+    exit 0
+fi
+
+
 # Patch crates to be on same versions
 mkdir -p $CARGO_HOME
+touch $CARGO_HOME/config.toml
 if [[ $PROD != "true" ]]; then
-    echo '[patch.crates-io]
+    echo '
+    [patch.crates-io]
     shuttle-service = { path = "/usr/src/shuttle/service" }
     shuttle-runtime = { path = "/usr/src/shuttle/runtime" }
 
@@ -18,8 +31,8 @@ if [[ $PROD != "true" ]]; then
     shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" }
     shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }
 
-    shuttle-axum = { path = "/usr/src/shuttle/services/shuttle-axum" }
     shuttle-actix-web = { path = "/usr/src/shuttle/services/shuttle-actix-web" }
+    shuttle-axum = { path = "/usr/src/shuttle/services/shuttle-axum" }
     shuttle-next = { path = "/usr/src/shuttle/services/shuttle-next" }
     shuttle-poem = { path = "/usr/src/shuttle/services/shuttle-poem" }
     shuttle-poise = { path = "/usr/src/shuttle/services/shuttle-poise" }
@@ -30,26 +43,14 @@ if [[ $PROD != "true" ]]; then
     shuttle-tide = { path = "/usr/src/shuttle/services/shuttle-tide" }
     shuttle-tower = { path = "/usr/src/shuttle/services/shuttle-tower" }
     shuttle-warp = { path = "/usr/src/shuttle/services/shuttle-warp" }' > $CARGO_HOME/config.toml
-else
-    touch $CARGO_HOME/config.toml
 fi
 
-# Install protoc since some users may need it
-ARCH="linux-x86_64" && \
-VERSION="22.2" && \
-curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \
-    unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \
-    rm -f "protoc-$VERSION-$ARCH.zip"
-
 # Add the wasm32-wasi target
 rustup target add wasm32-wasi
 
-# Install the shuttle runtime
-cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next
-
 while getopts "p," o; do
     case $o in
-        "p")
+        "p") # if panamax is used, the '-p' parameter is passed
             # Make future crates requests to our own mirror
             echo '
 [source.shuttle-crates-io-mirror]
@@ -62,12 +63,14 @@ replace-with = "shuttle-crates-io-mirror"' >> $CARGO_HOME/config.toml
     esac
 done
 
-# Prefetch crates.io index from our mirror
-# TODO: restore when we know how to prefetch from our mirror
-# cd /usr/src/shuttle/service
-# cargo fetch
-
 # Install common build tools for external crates
 # The image should already have these: https://github.com/docker-library/buildpack-deps/blob/65d69325ad741cea6dee20781c1faaab2e003d87/debian/buster/Dockerfile
 apt update
-apt install -y llvm-dev libclang-dev clang cmake
+apt install -y curl llvm-dev libclang-dev clang cmake
+
+# Install protoc since some users may need it
+ARCH="linux-x86_64" && \
+VERSION="22.2" && \
+curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \
+    unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \
+    rm -f "protoc-$VERSION-$ARCH.zip"
diff --git a/extras/postgres/Containerfile b/extras/postgres/Containerfile
@@ -2,8 +2,7 @@ ARG POSTGRES_TAG=
 
 FROM docker.io/postgres:${POSTGRES_TAG}
 
-RUN apt-get update &&\
-    apt-get install --yes curl python3 python3-aiohttp
+RUN apt update && apt install -y curl python3 python3-aiohttp
 
 COPY watch /usr/sbin/watch
 COPY shuttle-entrypoint.sh /usr/local/bin/shuttle-entrypoint.sh

diff --git a/gateway/prepare.sh b/gateway/prepare.sh
@@ -5,4 +5,9 @@
 # service might need some extra preparation steps for its final image         #
 ###############################################################################
 
+# Stuff that depends on local source files
+if [ "$1" = "--after-src" ]; then
+    exit 0
+fi
+
 # Nothing to prepare in container image here
diff --git a/provisioner/prepare.sh b/provisioner/prepare.sh
@@ -5,4 +5,9 @@
 # service might need some extra preparation steps for its final image         #
 ###############################################################################
 
+# Stuff that depends on local source files
+if [ "$1" = "--after-src" ]; then
+    exit 0
+fi
+
 # Nothing to prepare in container image here
diff --git a/service/src/builder.rs b/service/src/builder.rs
@@ -144,12 +144,9 @@ pub async fn clean_crate(project_path: &Path, release_mode: bool) -> anyhow::Res
     let project_path = project_path.to_owned();
     let manifest_path = project_path.join("Cargo.toml");
     if !manifest_path.exists() {
-        return Err(anyhow!("failed to read the Shuttle project manifest"));
-    }
-    let mut profile = "dev";
-    if release_mode {
-        profile = "release";
+        bail!("failed to read the Shuttle project manifest");
     }
+    let profile = if release_mode { "release" } else { "dev" };
     let output = tokio::process::Command::new("cargo")
         .arg("clean")
         .arg("--manifest-path")
@@ -221,6 +218,9 @@ async fn compile(
     tx: Sender<Message>,
 ) -> anyhow::Result<Vec<BuiltService>> {
     let manifest_path = project_path.join("Cargo.toml");
+    if !manifest_path.exists() {
+        bail!("failed to read the Shuttle project manifest");
+    }
     let target_path = target_path.into();
 
     let mut cargo = tokio::process::Command::new("cargo");
@@ -240,14 +240,13 @@ async fn compile(
         cargo.arg("--package").arg(package.name.clone());
     }
 
-    let mut profile = "debug";
-
-    if release_mode {
-        profile = "release";
+    let profile = if release_mode {
         cargo.arg("--profile").arg("release");
+        "release"
     } else {
         cargo.arg("--profile").arg("dev");
-    }
+        "debug"
+    };
 
     if wasm {
         cargo.arg("--target").arg("wasm32-wasi");
@@ -283,8 +282,7 @@ async fn compile(
                 target_path.clone(),
                 "wasm32-wasi".into(),
                 profile.into(),
-                #[allow(clippy::single_char_pattern)]
-                package.clone().name.replace("-", "_").into(),
+                package.name.replace('-', "_").into(),
             ]
             .iter()
             .collect();