feat: build our images with the release profile (#583)

* feat: build our images with the release profile

Currently our containerfile builds images with the default debug profile, which means when `get_runtime_executable` is called it will try to install from a checked out local version of `shuttle-runtime`.

* feat: add build arg for cargo profile

* refactor: update comment

* fix: broken codegen/axum test

* feat: install wasm32-wasi target in deploy.sh

* feat: build deps in `CARGO_PROFILE`

* feat: use closure to determine shuttle-runtime path

* fix: path to runtime in docker deployer, clippy

* fix: incorrect function for cargo home

* refactor: always use cargo_home for path to runtime in deployer

Containerfile

@@ -22,10 +22,12 @@ RUN cargo chef prepare --recipe-path recipe.json
 
 FROM shuttle-build AS builder
 COPY --from=planner /build/recipe.json recipe.json
-RUN cargo chef cook --recipe-path recipe.json
+RUN cargo chef cook $(if [ "$CARGO_PROFILE" = "release" ]; then echo --${CARGO_PROFILE}; fi) --recipe-path recipe.json
 COPY --from=cache /build .
 ARG folder
-RUN cargo build --bin shuttle-${folder}
+ARG CARGO_PROFILE
+# if CARGO_PROFILE is release, pass --release, else use default debug profile
+RUN cargo build --bin shuttle-${folder} $(if [ "$CARGO_PROFILE" = "release" ]; then echo --${CARGO_PROFILE}; fi)
 
 ARG RUSTUP_TOOLCHAIN
 FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common
@@ -43,7 +45,8 @@ FROM shuttle-common
 ARG folder
 COPY ${folder}/prepare.sh /prepare.sh
 RUN /prepare.sh
-COPY --from=builder /build/target/debug/shuttle-${folder} /usr/local/bin/service
+ARG CARGO_PROFILE
+COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-${folder} /usr/local/bin/service
 ARG RUSTUP_TOOLCHAIN
 ENV RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN}
 ENTRYPOINT ["/usr/local/bin/service"]

Makefile

@@ -49,6 +49,7 @@ CONTAINER_REGISTRY=public.ecr.aws/shuttle
 DD_ENV=production
 # make sure we only ever go to production with `--tls=enable`
 USE_TLS=enable
+CARGO_PROFILE=release
 else
 DOCKER_COMPOSE_FILES=-f docker-compose.yml -f docker-compose.dev.yml
 STACK?=shuttle-dev
@@ -57,6 +58,7 @@ DB_FQDN=db.unstable.shuttle.rs
 CONTAINER_REGISTRY=public.ecr.aws/shuttle-dev
 DD_ENV=unstable
 USE_TLS?=disable
+CARGO_PROFILE=debug
 endif
 
 POSTGRES_EXTRA_PATH?=./extras/postgres
@@ -112,6 +114,7 @@ shuttle-%: ${SRC} Cargo.lock
 	docker buildx build \
 	       --build-arg folder=$(*) \
 		   --build-arg RUSTUP_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) \
+		   --build-arg CARGO_PROFILE=$(CARGO_PROFILE) \
 	       --tag $(CONTAINER_REGISTRY)/$(*):$(COMMIT_SHA) \
 	       --tag $(CONTAINER_REGISTRY)/$(*):$(TAG) \
 	       --tag $(CONTAINER_REGISTRY)/$(*):latest \

cargo-shuttle/Cargo.toml

@@ -24,6 +24,7 @@ dirs = "4.0.0"
 flate2 = "1.0.25"
 futures = "0.3.25"
 git2 = "0.14.2"
+home = "0.5.4"
 headers = "0.3.8"
 indicatif = "0.17.2"
 ignore = "0.4.18"

cargo-shuttle/src/lib.rs

@@ -4,6 +4,7 @@ pub mod config;
 mod init;
 mod provisioner_server;
 
+use cargo::util::ToSemver;
 use shuttle_common::project::ProjectName;
 use shuttle_proto::runtime::{self, LoadRequest, StartRequest, SubscribeLogsRequest};
 use std::collections::HashMap;
@@ -40,6 +41,9 @@ use crate::args::{DeploymentCommand, ProjectCommand};
 use crate::client::Client;
 use crate::provisioner_server::LocalProvisioner;
 
+const VERSION: &str = env!("CARGO_PKG_VERSION");
+const MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
+
 pub struct Shuttle {
     ctx: RequestContext,
 }
@@ -410,11 +414,53 @@ impl Shuttle {
             Ipv4Addr::LOCALHOST.into(),
             run_args.port + 1,
         ));
+
+        let get_runtime_executable = || {
+            let runtime_path = home::cargo_home()
+                .expect("failed to find cargo home dir")
+                .join("bin/shuttle-runtime");
+
+            if cfg!(debug_assertions) {
+                // Canonicalized path to shuttle-runtime for dev to work on windows
+                let path = std::fs::canonicalize(format!("{MANIFEST_DIR}/../runtime"))
+                    .expect("path to shuttle-runtime does not exist or is invalid");
+
+                std::process::Command::new("cargo")
+                    .arg("install")
+                    .arg("shuttle-runtime")
+                    .arg("--path")
+                    .arg(path)
+                    .output()
+                    .expect("failed to install the shuttle runtime");
+            } else {
+                // If the version of cargo-shuttle is different from shuttle-runtime,
+                // or it isn't installed, try to install shuttle-runtime from the production
+                // branch.
+                if let Err(err) = check_version(&runtime_path) {
+                    trace!("{}", err);
+
+                    trace!("installing shuttle-runtime");
+                    std::process::Command::new("cargo")
+                        .arg("install")
+                        .arg("shuttle-runtime")
+                        .arg("--git")
+                        .arg("https://github.com/shuttle-hq/shuttle")
+                        .arg("--branch")
+                        .arg("production")
+                        .output()
+                        .expect("failed to install the shuttle runtime");
+                };
+            };
+
+            runtime_path
+        };
+
         let (mut runtime, mut runtime_client) = runtime::start(
             is_wasm,
             runtime::StorageManagerType::WorkingDir(working_directory.to_path_buf()),
             &format!("http://localhost:{}", run_args.port + 1),
             run_args.port + 2,
+            get_runtime_executable,
         )
         .await
         .map_err(|err| {
@@ -772,6 +818,40 @@ impl Shuttle {
     }
 }
 
+fn check_version(runtime_path: &Path) -> Result<()> {
+    let valid_version = VERSION.to_semver().unwrap();
+
+    if !runtime_path.try_exists()? {
+        bail!("shuttle-runtime is not installed");
+    }
+
+    // Get runtime version from shuttle-runtime cli
+    let runtime_version = std::process::Command::new("cargo")
+        .arg("shuttle-runtime")
+        .arg("--version")
+        .output()
+        .context("failed to check the shuttle-runtime version")?
+        .stdout;
+
+    // Parse the version, splitting the version from the name and
+    // and pass it to `to_semver()`.
+    let runtime_version = std::str::from_utf8(&runtime_version)
+        .expect("shuttle-runtime version should be valid utf8")
+        .split_once(' ')
+        .expect("shuttle-runtime version should be in the `name version` format")
+        .1
+        .to_semver()
+        .context("failed to convert runtime version to semver")?;
+
+    if runtime_version == valid_version {
+        Ok(())
+    } else {
+        Err(anyhow!(
+            "shuttle-runtime and cargo-shuttle are not the same version"
+        ))
+    }
+}
+
 pub enum CommandOutcome {
     Ok,
     DeploymentFailure,

codegen/src/next/mod.rs

@@ -307,7 +307,9 @@ pub(crate) fn wasi_bindings(app: App) -> proc_macro2::TokenStream {
             let (parts, mut body) = res.into_parts();
 
             // wrap and serialize response parts as rmp
-            let response_parts = shuttle_next::ResponseWrapper::from(parts).into_rmp();
+            let response_parts = shuttle_next::ResponseWrapper::from(parts)
+                .into_rmp()
+                .expect("failed to serialize response parts");
 
             // write response parts
             parts_fd.write_all(&response_parts).unwrap();

deployer/Cargo.toml

@@ -19,6 +19,7 @@ crossbeam-channel = "0.5.6"
 flate2 = "1.0.25"
 fqdn = "0.2.3"
 futures = "0.3.25"
+home = "0.5.4"
 hyper = { version = "0.14.23", features = ["client", "http1", "http2", "tcp" ] }
 # not great, but waiting for WebSocket changes to be merged
 hyper-reverse-proxy = { git = "https://github.com/chesedo/hyper-reverse-proxy", branch = "master" }

deployer/prepare.sh

@@ -15,6 +15,9 @@ shuttle-shared-db = { path = "/usr/src/shuttle/resources/shared-db" }
 shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" }
 shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' > $CARGO_HOME/config.toml
 
+# Add the wasm32-wasi target
+rustup target add wasm32-wasi
+
 # Install the shuttle runtime
 cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" 
 

deployer/src/runtime_manager.rs

@@ -8,6 +8,8 @@ use tracing::{info, instrument, trace};
 
 use crate::deployment::deploy_layer;
 
+const MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
+
 #[derive(Clone)]
 pub struct RuntimeManager {
     legacy: Option<RuntimeClient<Channel>>,
@@ -80,11 +82,39 @@ impl RuntimeManager {
             Ok(runtime_client)
         } else {
             trace!("making new client");
+
+            let get_runtime_executable = || {
+                if cfg!(debug_assertions) {
+                    // If we're running deployer natively, install shuttle-runtime using the
+                    // version of runtime from the calling repo.
+                    let path = std::fs::canonicalize(format!("{MANIFEST_DIR}/../runtime"));
+
+                    // The path will not be valid if we are in a deployer container, in which
+                    // case we don't try to install and use the one installed in deploy.sh.
+                    if let Ok(path) = path {
+                        std::process::Command::new("cargo")
+                            .arg("install")
+                            .arg("shuttle-runtime")
+                            .arg("--path")
+                            .arg(path)
+                            .output()
+                            .expect("failed to install the local version of shuttle-runtime");
+                    }
+                }
+
+                // If we're in a deployer built with the containerfile, the runtime will have
+                // been installed in deploy.sh.
+                home::cargo_home()
+                    .expect("failed to find path to cargo home")
+                    .join("bin/shuttle-runtime")
+            };
+
             let (process, runtime_client) = runtime::start(
                 is_next,
                 runtime::StorageManagerType::Artifacts(artifacts_path),
                 provisioner_address,
                 port,
+                get_runtime_executable,
             )
             .await
             .context("failed to start shuttle runtime")?;

proto/src/lib.rs

@@ -98,7 +98,6 @@ pub mod runtime {
     use std::{
         convert::TryFrom,
         path::PathBuf,
-        process::Command,
         time::{Duration, SystemTime},
     };
 
@@ -246,6 +245,7 @@ pub mod runtime {
         storage_manager_type: StorageManagerType,
         provisioner_address: &str,
         port: u16,
+        get_runtime_executable: impl FnOnce() -> PathBuf,
     ) -> anyhow::Result<(process::Child, runtime_client::RuntimeClient<Channel>)> {
         let runtime_flag = if wasm { "--axum" } else { "--legacy" };
 
@@ -286,41 +286,4 @@ pub mod runtime {
 
         Ok((runtime, runtime_client))
     }
-
-    fn get_runtime_executable() -> PathBuf {
-        // When this library is compiled in debug mode with `cargo run --bin cargo-shuttle`,
-        // install the checked-out local version of `shuttle-runtime
-        if cfg!(debug_assertions) {
-            // Path to cargo-shuttle
-            let manifest_dir = env!("CARGO_MANIFEST_DIR");
-
-            // Canonicalized path to shuttle-runtime
-            let path = std::fs::canonicalize(format!("{manifest_dir}/../runtime"))
-                .expect("failed to canonicalize path to runtime");
-
-            Command::new("cargo")
-                .arg("install")
-                .arg("shuttle-runtime")
-                .arg("--path")
-                .arg(path)
-                .output()
-                .expect("failed to install the shuttle runtime");
-        // When this library is compiled in release mode with `cargo install cargo-shuttle`,
-        // install the latest released `shuttle-runtime`
-        } else {
-            Command::new("cargo")
-                .arg("install")
-                .arg("shuttle-runtime")
-                .arg("--git")
-                .arg("https://github.com/shuttle-hq/shuttle")
-                .arg("--branch")
-                .arg("production")
-                .output()
-                .expect("failed to install the shuttle runtime");
-        }
-
-        let cargo_home = home::cargo_home().expect("failed to find cargo home directory");
-
-        cargo_home.join("bin/shuttle-runtime")
-    }
 }