Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into chore/workspace-eve…
Browse files Browse the repository at this point in the history
…rything
  • Loading branch information
jonaro00 committed Jun 26, 2023
2 parents d2d945d + 947d6a7 commit a05f064
Show file tree
Hide file tree
Showing 9 changed files with 139 additions and 105 deletions.
94 changes: 47 additions & 47 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ commands:
- run:
name: Install sccache
command: |
export SCCACHE_VERSION='v0.4.2'
export SCCACHE_VERSION='v0.5.3'
ls ~/.cargo/bin/sccache || curl -L https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz | tar -xOz sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl/sccache > ~/.cargo/bin/sccache && chmod +x ~/.cargo/bin/sccache
# This configures Rust to use sccache.
echo 'export "RUSTC_WRAPPER"="sccache"' >> $BASH_ENV
Expand Down Expand Up @@ -304,7 +304,7 @@ jobs:
default: false
steps:
- checkout
- run:
- run:
name: Set git tag in bash_env
command: |
echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV
Expand Down Expand Up @@ -347,40 +347,40 @@ jobs:
type: boolean
default: false
steps:
- checkout
- run:
name: Set git tag in bash_env
command: |
echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV
- add_ssh_keys:
fingerprints:
- << parameters.ssh-fingerprint >>
- run:
name: Generate ssh config
command: ./.circleci/<< parameters.ssh-config-script >>
- run:
name: Deploy images
command: |
DOCKER_HOST=ssh://ec2-user@master.<< parameters.ssh-host >> USE_TLS=enable PROD=<< parameters.production >> DD_API_KEY=$DD_API_KEY \
POSTGRES_PASSWORD=${<< parameters.postgres-password >>} \
MONGO_INITDB_ROOT_PASSWORD=${<< parameters.mongodb-password >>} \
TAG=$TAG \
make deploy
- when:
condition: << parameters.production >>
steps:
- run:
name: Pull new deployer image on prod
command: |
ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle/deployer:$TAG"
- when:
condition:
not: << parameters.production >>
steps:
- run:
name: Pull new deployer image on dev
command: |
ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle-dev/deployer:$TAG"
- checkout
- run:
name: Set git tag in bash_env
command: |
echo TAG=$(git describe --tags --abbrev=0) >> $BASH_ENV
- add_ssh_keys:
fingerprints:
- << parameters.ssh-fingerprint >>
- run:
name: Generate ssh config
command: ./.circleci/<< parameters.ssh-config-script >>
- run:
name: Deploy images
command: |
DOCKER_HOST=ssh://ec2-user@master.<< parameters.ssh-host >> USE_TLS=enable PROD=<< parameters.production >> DD_API_KEY=$DD_API_KEY \
POSTGRES_PASSWORD=${<< parameters.postgres-password >>} \
MONGO_INITDB_ROOT_PASSWORD=${<< parameters.mongodb-password >>} \
TAG=$TAG \
make deploy
- when:
condition: << parameters.production >>
steps:
- run:
name: Pull new deployer image on prod
command: |
ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle/deployer:$TAG"
- when:
condition:
not: << parameters.production >>
steps:
- run:
name: Pull new deployer image on dev
command: |
ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle-dev/deployer:$TAG"
build-binaries-linux:
machine:
image: << parameters.image >>
Expand Down Expand Up @@ -487,7 +487,7 @@ jobs:
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
- run:
name: "Install Shuttle"
command: cargo install cargo-shuttle --path ./cargo-shuttle
command: cargo install cargo-shuttle --path ./cargo-shuttle
- run: cargo shuttle --version
- run:
name: Login
Expand Down Expand Up @@ -518,7 +518,7 @@ jobs:
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
- run:
name: "Install Shuttle"
command: cargo install cargo-shuttle --path ./cargo-shuttle
command: cargo install cargo-shuttle --path ./cargo-shuttle
- run: cargo shuttle --version
- run:
name: Login
Expand Down Expand Up @@ -563,7 +563,7 @@ jobs:
C:\rustup-init.exe -y --default-toolchain 1.70.0 --target x86_64-pc-windows-msvc
- run:
name: "Install Shuttle"
command: ..\.cargo\bin\cargo.exe install cargo-shuttle --path ./cargo-shuttle
command: ..\.cargo\bin\cargo.exe install cargo-shuttle --path ./cargo-shuttle
- run: ..\.cargo\bin\cargo.exe shuttle --version
- run:
name: Login
Expand Down Expand Up @@ -827,13 +827,13 @@ workflows:
- publish-crates:
matrix:
parameters:
path:
path:
[
"resources/aws-rds",
"resources/shared-db",
"resources/secrets",
"resources/persist",
"resources/static-folder"
"resources/static-folder",
]
name: publish-<< matrix.path >>
requires:
Expand All @@ -852,20 +852,20 @@ workflows:
- publish-crates:
matrix:
parameters:
path:
path:
[
"services/shuttle-actix-web",
"services/shuttle-axum",
"services/shuttle-next",
"services/shuttle-poem",
"services/shuttle-actix-web",
"services/shuttle-axum",
"services/shuttle-next",
"services/shuttle-poem",
"services/shuttle-poise",
"services/shuttle-rocket",
"services/shuttle-salvo",
"services/shuttle-serenity",
"services/shuttle-thruster",
"services/shuttle-tide",
"services/shuttle-tower",
"services/shuttle-warp"
"services/shuttle-warp",
]
name: publish-<< matrix.path >>
requires:
Expand Down
60 changes: 38 additions & 22 deletions Containerfile
Original file line number Diff line number Diff line change
@@ -1,47 +1,63 @@
#syntax=docker/dockerfile-upstream:1.4.0-rc1
#syntax=docker/dockerfile-upstream:1.4


# Base image for builds and cache
ARG RUSTUP_TOOLCHAIN
FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-build
RUN apt-get update &&\
apt-get install -y curl

RUN cargo install cargo-chef
RUN cargo install cargo-chef --locked
WORKDIR /build


# Stores source cache
FROM shuttle-build as cache
ARG CARGO_PROFILE
WORKDIR /src
COPY . .
RUN find ${SRC_CRATES} \( -name "*.proto" -or -name "*.rs" -or -name "*.toml" -or -name "Cargo.lock" -or -name "README.md" -or -name "*.sql" \) -type f -exec install -D \{\} /build/\{\} \;
# This is used to carry over in the docker images any *.pem files from shuttle root directory, to be used for TLS testing, as described
# here in the admin README.md.
RUN [ "$CARGO_PROFILE" != "release" ] && find ${SRC_CRATES} -name "*.pem" -type f -exec install -D \{\} /build/\{\} \;
# This is used to carry over in the docker images any *.pem files from shuttle root directory,
# to be used for TLS testing, as described here in the admin README.md.
RUN [ "$CARGO_PROFILE" != "release" ] && \
find ${SRC_CRATES} -name "*.pem" -type f -exec install -D \{\} /build/\{\} \;


# Stores cargo chef recipe
FROM shuttle-build AS planner
COPY --from=cache /build .
RUN cargo chef prepare --recipe-path recipe.json


# Builds crate according to cargo chef recipe
FROM shuttle-build AS builder
COPY --from=planner /build/recipe.json recipe.json
ARG CARGO_PROFILE
RUN cargo chef cook $(if [ "$CARGO_PROFILE" = "release" ]; then echo --${CARGO_PROFILE}; fi) --recipe-path recipe.json
COPY --from=cache /build .
ARG folder
# if CARGO_PROFILE is release, pass --release, else use default debug profile
RUN cargo build --bin shuttle-${folder} $(if [ "$CARGO_PROFILE" = "release" ]; then echo --${CARGO_PROFILE}; fi)

ARG RUSTUP_TOOLCHAIN
FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common
RUN rustup component add rust-src
COPY --from=planner /build/recipe.json recipe.json
RUN cargo chef cook \
# if CARGO_PROFILE is release, pass --release, else use default debug profile
$(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi) \
--recipe-path recipe.json
COPY --from=cache /build .
RUN cargo build --bin shuttle-${folder} \
$(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi)

COPY --from=cache /build/ /usr/src/shuttle/

FROM shuttle-common
# The final image for this "shuttle-..." crate
ARG RUSTUP_TOOLCHAIN
FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-crate
ARG folder
ARG prepare_args
# used as env variable in prepare script
ARG PROD
COPY ${folder}/prepare.sh /prepare.sh
RUN /prepare.sh "${prepare_args}"
ARG CARGO_PROFILE
COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-${folder} /usr/local/bin/service
ARG RUSTUP_TOOLCHAIN
ENV RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN}

COPY ${folder}/prepare.sh /prepare.sh
RUN /prepare.sh "${prepare_args}"

COPY --from=cache /build /usr/src/shuttle/

# Any prepare steps that depend on the COPY from src cache
RUN /prepare.sh --after-src "${prepare_args}"

COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-${folder} /usr/local/bin/service
ENTRYPOINT ["/usr/local/bin/service"]
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ which will automatically install the correct target for your system.
To install with `cargo-binstall`, run:

```sh
cargo install binstall
cargo binstall cargo-shuttle
```

Expand All @@ -76,6 +77,8 @@ Although a bit slower, you can also install directly with cargo:
cargo install cargo-shuttle
```

*If installing binstall or cargo-shuttle fails, try adding `--locked` to the install command.*

After installing, log in with:

```sh
Expand Down
5 changes: 5 additions & 0 deletions auth/prepare.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,9 @@
# service might need some extra preparation steps for its final image #
###############################################################################

# Stuff that depends on local source files
if [ "$1" = "--after-src" ]; then
exit 0
fi

# Nothing to prepare in container image here
47 changes: 25 additions & 22 deletions deployer/prepare.sh
Original file line number Diff line number Diff line change
@@ -1,14 +1,27 @@
#!/usr/bin/env sh
#!/bin/bash

###############################################################################
# This file is used by our common Containerfile incase the container for this #
# service might need some extra preparation steps for its final image #
###############################################################################


# Stuff that depends on local source files
if [ "$1" = "--after-src" ]; then

# Install the shuttle runtime
cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next

exit 0
fi


# Patch crates to be on same versions
mkdir -p $CARGO_HOME
touch $CARGO_HOME/config.toml
if [[ $PROD != "true" ]]; then
echo '[patch.crates-io]
echo '
[patch.crates-io]
shuttle-codegen = { path = "/usr/src/shuttle/codegen" }
shuttle-common = { path = "/usr/src/shuttle/common" }
shuttle-proto = { path = "/usr/src/shuttle/proto" }
Expand All @@ -21,8 +34,8 @@ if [[ $PROD != "true" ]]; then
shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" }
shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }
shuttle-axum = { path = "/usr/src/shuttle/services/shuttle-axum" }
shuttle-actix-web = { path = "/usr/src/shuttle/services/shuttle-actix-web" }
shuttle-axum = { path = "/usr/src/shuttle/services/shuttle-axum" }
shuttle-next = { path = "/usr/src/shuttle/services/shuttle-next" }
shuttle-poem = { path = "/usr/src/shuttle/services/shuttle-poem" }
shuttle-poise = { path = "/usr/src/shuttle/services/shuttle-poise" }
Expand All @@ -33,26 +46,14 @@ if [[ $PROD != "true" ]]; then
shuttle-tide = { path = "/usr/src/shuttle/services/shuttle-tide" }
shuttle-tower = { path = "/usr/src/shuttle/services/shuttle-tower" }
shuttle-warp = { path = "/usr/src/shuttle/services/shuttle-warp" }' > $CARGO_HOME/config.toml
else
touch $CARGO_HOME/config.toml
fi

# Install protoc since some users may need it
ARCH="linux-x86_64" && \
VERSION="22.2" && \
curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \
unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \
rm -f "protoc-$VERSION-$ARCH.zip"

# Add the wasm32-wasi target
rustup target add wasm32-wasi

# Install the shuttle runtime
cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next

while getopts "p," o; do
case $o in
"p")
"p") # if panamax is used, the '-p' parameter is passed
# Make future crates requests to our own mirror
echo '
[source.shuttle-crates-io-mirror]
Expand All @@ -65,12 +66,14 @@ replace-with = "shuttle-crates-io-mirror"' >> $CARGO_HOME/config.toml
esac
done

# Prefetch crates.io index from our mirror
# TODO: restore when we know how to prefetch from our mirror
# cd /usr/src/shuttle/service
# cargo fetch

# Install common build tools for external crates
# The image should already have these: https://github.com/docker-library/buildpack-deps/blob/65d69325ad741cea6dee20781c1faaab2e003d87/debian/buster/Dockerfile
apt update
apt install -y llvm-dev libclang-dev clang cmake
apt install -y curl llvm-dev libclang-dev clang cmake

# Install protoc since some users may need it
ARCH="linux-x86_64" && \
VERSION="22.2" && \
curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \
unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \
rm -f "protoc-$VERSION-$ARCH.zip"
3 changes: 1 addition & 2 deletions extras/postgres/Containerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@ ARG POSTGRES_TAG=

FROM docker.io/postgres:${POSTGRES_TAG}

RUN apt-get update &&\
apt-get install --yes curl python3 python3-aiohttp
RUN apt update && apt install -y curl python3 python3-aiohttp

COPY watch /usr/sbin/watch
COPY shuttle-entrypoint.sh /usr/local/bin/shuttle-entrypoint.sh
Expand Down
5 changes: 5 additions & 0 deletions gateway/prepare.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,9 @@
# service might need some extra preparation steps for its final image #
###############################################################################

# Stuff that depends on local source files
if [ "$1" = "--after-src" ]; then
exit 0
fi

# Nothing to prepare in container image here
5 changes: 5 additions & 0 deletions provisioner/prepare.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,9 @@
# service might need some extra preparation steps for its final image #
###############################################################################

# Stuff that depends on local source files
if [ "$1" = "--after-src" ]; then
exit 0
fi

# Nothing to prepare in container image here
Loading

0 comments on commit a05f064

Please sign in to comment.