diff --git a/.circleci/config.yml b/.circleci/config.yml index 9211a8b86..925bbc209 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,6 +89,7 @@ commands: shuttle-aws-rds = { path = "$PWD/resources/aws-rds" } shuttle-persist = { path = "$PWD/resources/persist" } shuttle-next = { path = "$PWD/next" } + shuttle-runtime = { path = "$PWD/runtime" } shuttle-shared-db = { path = "$PWD/resources/shared-db" } shuttle-secrets = { path = "$PWD/resources/secrets" } shuttle-static-folder = { path = "$PWD/resources/static-folder" } diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 4890433cd..730d11360 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -449,7 +449,7 @@ impl Shuttle { let service_name = self.ctx.project_name().to_string(); - let (is_wasm, bin_path) = match runtime { + let (is_wasm, executable_path) = match runtime { Runtime::Next(path) => (true, path), Runtime::Legacy(path) => (false, path), }; @@ -479,6 +479,8 @@ impl Shuttle { .arg(path) .arg("--bin") .arg("shuttle-next") + .arg("--features") + .arg("next") .output() .expect("failed to install the shuttle runtime"); } else { @@ -499,6 +501,8 @@ impl Shuttle { .arg("https://github.com/shuttle-hq/shuttle") .arg("--branch") .arg("production") + .arg("--features") + .arg("next") .output() .expect("failed to install the shuttle runtime"); }; @@ -506,7 +510,7 @@ impl Shuttle { runtime_path } else { - bin_path.clone() + executable_path.clone() } }; @@ -525,7 +529,7 @@ impl Shuttle { })?; let load_request = tonic::Request::new(LoadRequest { - path: bin_path + path: executable_path .into_os_string() .into_string() .expect("to convert path to string"), diff --git a/codegen/src/shuttle_main/mod.rs b/codegen/src/shuttle_main/mod.rs index 771e751cc..97ca75813 100644 --- a/codegen/src/shuttle_main/mod.rs +++ b/codegen/src/shuttle_main/mod.rs @@ -239,8 +239,8 @@ impl ToTokens for Loader { }; let loader = quote! { - async fn loader( - mut #factory_ident: shuttle_runtime::ProvisionerFactory, + async fn loader( + mut #factory_ident: shuttle_runtime::ProvisionerFactory, logger: shuttle_runtime::Logger, ) -> #return_type { use shuttle_service::Context; @@ -299,8 +299,8 @@ mod tests { let actual = quote!(#input); let expected = quote! { - async fn loader( - mut _factory: shuttle_runtime::ProvisionerFactory, + async fn loader( + mut _factory: shuttle_runtime::ProvisionerFactory, logger: shuttle_runtime::Logger, ) -> ShuttleSimple { use shuttle_service::Context; @@ -379,8 +379,8 @@ mod tests { let actual = quote!(#input); let expected = quote! { - async fn loader( - mut factory: shuttle_runtime::ProvisionerFactory, + async fn loader( + mut factory: shuttle_runtime::ProvisionerFactory, logger: shuttle_runtime::Logger, ) -> ShuttleComplex { use shuttle_service::Context; @@ -494,8 +494,8 @@ mod tests { let actual = quote!(#input); let expected = quote! { - async fn loader( - mut factory: shuttle_runtime::ProvisionerFactory, + async fn loader( + mut factory: shuttle_runtime::ProvisionerFactory, logger: shuttle_runtime::Logger, ) -> ShuttleComplex { use shuttle_service::Context; diff --git a/common/src/storage_manager.rs b/common/src/storage_manager.rs index f2270243a..22ac50c4e 100644 --- a/common/src/storage_manager.rs +++ b/common/src/storage_manager.rs @@ -2,14 +2,14 @@ use std::{fs, io, path::PathBuf}; use uuid::Uuid; -pub trait StorageManager: Clone + Sync + Send { +pub trait StorageManager: Sync + Send { /// Path for a specific service build files - fn service_build_path>(&self, service_name: S) -> Result; + fn service_build_path(&self, service_name: &str) -> Result; /// Path to folder for storing deployment files - fn deployment_storage_path>( + fn deployment_storage_path( &self, - service_name: S, + service_name: &str, deployment_id: &Uuid, ) -> Result; } @@ -33,19 +33,19 @@ impl ArtifactsStorageManager { Ok(builds_path) } - /// The directory in which compiled '.so' files are stored. - pub fn libraries_path(&self) -> Result { - let libs_path = self.artifacts_path.join("shuttle-libs"); - fs::create_dir_all(&libs_path)?; + /// The directory in which compiled executables are stored. + pub fn executables_path(&self) -> Result { + let executables_path = self.artifacts_path.join("shuttle-executables"); + fs::create_dir_all(&executables_path)?; - Ok(libs_path) + Ok(executables_path) } - /// Path to `.so` for a service - pub fn deployment_library_path(&self, deployment_id: &Uuid) -> Result { - let library_path = self.libraries_path()?.join(deployment_id.to_string()); + /// Path to executable for a service + pub fn deployment_executable_path(&self, deployment_id: &Uuid) -> Result { + let executable_path = self.executables_path()?.join(deployment_id.to_string()); - Ok(library_path) + Ok(executable_path) } /// Path of the directory to store user files @@ -58,21 +58,21 @@ impl ArtifactsStorageManager { } impl StorageManager for ArtifactsStorageManager { - fn service_build_path>(&self, service_name: S) -> Result { - let builds_path = self.builds_path()?.join(service_name.as_ref()); + fn service_build_path(&self, service_name: &str) -> Result { + let builds_path = self.builds_path()?.join(service_name); fs::create_dir_all(&builds_path)?; Ok(builds_path) } - fn deployment_storage_path>( + fn deployment_storage_path( &self, - service_name: S, + service_name: &str, deployment_id: &Uuid, ) -> Result { let storage_path = self .storage_path()? - .join(service_name.as_ref()) + .join(service_name) .join(deployment_id.to_string()); fs::create_dir_all(&storage_path)?; @@ -93,13 +93,13 @@ impl WorkingDirStorageManager { } impl StorageManager for WorkingDirStorageManager { - fn service_build_path>(&self, _service_name: S) -> Result { + fn service_build_path(&self, _service_name: &str) -> Result { Ok(self.working_dir.clone()) } - fn deployment_storage_path>( + fn deployment_storage_path( &self, - _service_name: S, + _service_name: &str, _deployment_id: &Uuid, ) -> Result { Ok(self.working_dir.clone()) diff --git a/deployer/prepare.sh b/deployer/prepare.sh index 6fc385470..882f8d320 100755 --- a/deployer/prepare.sh +++ b/deployer/prepare.sh @@ -9,17 +9,18 @@ mkdir -p $CARGO_HOME; \ echo '[patch.crates-io] shuttle-service = { path = "/usr/src/shuttle/service" } +shuttle-runtime = { path = "/usr/src/shuttle/runtime" } shuttle-aws-rds = { path = "/usr/src/shuttle/resources/aws-rds" } shuttle-persist = { path = "/usr/src/shuttle/resources/persist" } shuttle-shared-db = { path = "/usr/src/shuttle/resources/shared-db" } shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" } shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' > $CARGO_HOME/config.toml -# + # Add the wasm32-wasi target rustup target add wasm32-wasi # Install the shuttle runtime -cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" +cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next while getopts "p," o; do case $o in diff --git a/deployer/src/deployment/queue.rs b/deployer/src/deployment/queue.rs index cec7e0649..1c7263f3d 100644 --- a/deployer/src/deployment/queue.rs +++ b/deployer/src/deployment/queue.rs @@ -224,9 +224,9 @@ impl Queued { run_pre_deploy_tests(&project_path, tx).await?; } - info!("Moving built library"); + info!("Moving built executable"); - store_lib(&storage_manager, &runtime, &self.id).await?; + store_executable(&storage_manager, &runtime, &self.id).await?; let is_next = matches!(runtime, Runtime::Next(_)); @@ -396,21 +396,22 @@ async fn run_pre_deploy_tests( } } -/// Store 'so' file in the libs folder +/// This will store the path to the executable for each runtime, which will be the users project with +/// an embedded runtime for legacy, and a .wasm file for shuttle-next. #[instrument(skip(storage_manager, runtime, id))] -async fn store_lib( +async fn store_executable( storage_manager: &ArtifactsStorageManager, runtime: &Runtime, id: &Uuid, ) -> Result<()> { - let so_path = match runtime { + let executable_path = match runtime { Runtime::Next(path) => path, Runtime::Legacy(path) => path, }; - let new_so_path = storage_manager.deployment_library_path(id)?; + let new_executable_path = storage_manager.deployment_executable_path(id)?; - fs::rename(so_path, new_so_path).await?; + fs::rename(executable_path, new_executable_path).await?; Ok(()) } @@ -550,30 +551,34 @@ ff0e55bda1ff01000000000000000000e0079c01ff12a55500280000", } #[tokio::test] - async fn store_lib() { - let libs_dir = Builder::new().prefix("lib-store").tempdir().unwrap(); - let libs_p = libs_dir.path(); - let storage_manager = ArtifactsStorageManager::new(libs_p.to_path_buf()); + async fn store_executable() { + let executables_dir = Builder::new().prefix("executable-store").tempdir().unwrap(); + let executables_p = executables_dir.path(); + let storage_manager = ArtifactsStorageManager::new(executables_p.to_path_buf()); let build_p = storage_manager.builds_path().unwrap(); - let so_path = build_p.join("xyz.so"); - let runtime = Runtime::Legacy(so_path.clone()); + let executable_path = build_p.join("xyz"); + let runtime = Runtime::Legacy(executable_path.clone()); let id = Uuid::new_v4(); - fs::write(&so_path, "barfoo").await.unwrap(); + fs::write(&executable_path, "barfoo").await.unwrap(); - super::store_lib(&storage_manager, &runtime, &id) + super::store_executable(&storage_manager, &runtime, &id) .await .unwrap(); - // Old '.so' file gone? - assert!(!so_path.exists()); + // Old executable file gone? + assert!(!executable_path.exists()); assert_eq!( - fs::read_to_string(libs_p.join("shuttle-libs").join(id.to_string())) - .await - .unwrap(), + fs::read_to_string( + executables_p + .join("shuttle-executables") + .join(id.to_string()) + ) + .await + .unwrap(), "barfoo" ); } diff --git a/deployer/src/deployment/run.rs b/deployer/src/deployment/run.rs index f814766a2..4df39521c 100644 --- a/deployer/src/deployment/run.rs +++ b/deployer/src/deployment/run.rs @@ -182,7 +182,10 @@ impl Built { kill_old_deployments: impl futures::Future>, cleanup: impl FnOnce(SubscribeStopResponse) + Send + 'static, ) -> Result<()> { - let so_path = storage_manager.deployment_library_path(&self.id)?; + // For legacy this is the path to the users project with an embedded runtime. + // For shuttle-next this is the path to the compiled .wasm file, which will be + // used in the load request. + let executable_path = storage_manager.deployment_executable_path(&self.id)?; let port = match pick_unused_port() { Some(port) => port, @@ -194,10 +197,18 @@ impl Built { }; let address = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port); + + let legacy_runtime_path = if self.is_next { + // The runtime client for next is the installed shuttle-next bin + None + } else { + Some(executable_path.clone()) + }; + let runtime_client = runtime_manager .lock() .await - .get_runtime_client(self.is_next) + .get_runtime_client(legacy_runtime_path.clone()) .await .map_err(Error::Runtime)?; @@ -208,7 +219,7 @@ impl Built { load( self.service_name.clone(), self.service_id, - so_path, + executable_path.clone(), secret_getter, runtime_client.clone(), ) @@ -230,13 +241,13 @@ impl Built { async fn load( service_name: String, service_id: Uuid, - so_path: PathBuf, + executable_path: PathBuf, secret_getter: impl SecretGetter, mut runtime_client: RuntimeClient, ) -> Result<()> { info!( "loading project from: {}", - so_path + executable_path .clone() .into_os_string() .into_string() @@ -252,7 +263,10 @@ async fn load( let secrets = HashMap::from_iter(secrets); let load_request = tonic::Request::new(LoadRequest { - path: so_path.into_os_string().into_string().unwrap_or_default(), + path: executable_path + .into_os_string() + .into_string() + .unwrap_or_default(), service_name: service_name.clone(), secrets, }); @@ -662,10 +676,9 @@ mod tests { let id = Uuid::new_v4(); let so_path = crate_dir.join("target/release").join(lib_name); let storage_manager = get_storage_manager(); - let new_so_path = storage_manager.deployment_library_path(&id).unwrap(); + let new_so_path = storage_manager.deployment_executable_path(&id).unwrap(); std::fs::copy(so_path, new_so_path).unwrap(); - ( Built { id, diff --git a/deployer/src/deployment/storage_manager.rs b/deployer/src/deployment/storage_manager.rs deleted file mode 100644 index 5a5fa1300..000000000 --- a/deployer/src/deployment/storage_manager.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::{fs, io, path::PathBuf}; - -use uuid::Uuid; - -/// Manager to take care of directories for storing project, services and deployment files -#[derive(Clone)] -pub struct StorageManager { - artifacts_path: PathBuf, -} - -impl StorageManager { - pub fn new(artifacts_path: PathBuf) -> Self { - Self { artifacts_path } - } - - /// Path of the directory that contains extracted service Cargo projects. - pub fn builds_path(&self) -> Result { - let builds_path = self.artifacts_path.join("shuttle-builds"); - fs::create_dir_all(&builds_path)?; - - Ok(builds_path) - } - - /// Path for a specific service - pub fn service_build_path>(&self, service_name: S) -> Result { - let builds_path = self.builds_path()?.join(service_name.as_ref()); - fs::create_dir_all(&builds_path)?; - - Ok(builds_path) - } - - /// The directory in which compiled '.so' files are stored. - pub fn libraries_path(&self) -> Result { - let libs_path = self.artifacts_path.join("shuttle-libs"); - fs::create_dir_all(&libs_path)?; - - Ok(libs_path) - } - - /// Path to `.so` for a service - pub fn deployment_library_path(&self, deployment_id: &Uuid) -> Result { - let library_path = self.libraries_path()?.join(deployment_id.to_string()); - - Ok(library_path) - } - - /// Path of the directory to store user files - pub fn storage_path(&self) -> Result { - let storage_path = self.artifacts_path.join("shuttle-storage"); - fs::create_dir_all(&storage_path)?; - - Ok(storage_path) - } - - /// Path to folder for storing deployment files - pub fn deployment_storage_path>( - &self, - service_name: S, - deployment_id: &Uuid, - ) -> Result { - let storage_path = self - .storage_path()? - .join(service_name.as_ref()) - .join(deployment_id.to_string()); - fs::create_dir_all(&storage_path)?; - - Ok(storage_path) - } -} diff --git a/deployer/src/handlers/mod.rs b/deployer/src/handlers/mod.rs index 2e8e18237..c7a01f1c3 100644 --- a/deployer/src/handlers/mod.rs +++ b/deployer/src/handlers/mod.rs @@ -405,7 +405,7 @@ async fn post_clean( ) -> Result>> { let project_path = deployment_manager .storage_manager() - .service_build_path(project_name) + .service_build_path(&project_name) .map_err(anyhow::Error::new)?; let lines = clean_crate(&project_path, true)?; diff --git a/deployer/src/lib.rs b/deployer/src/lib.rs index 347ba254f..dc689820c 100644 --- a/deployer/src/lib.rs +++ b/deployer/src/lib.rs @@ -65,12 +65,11 @@ pub async fn start( args.project, ) .await; - let make_service = router.into_make_service(); info!(address=%args.api_address, "Binding to and listening at address"); axum::Server::bind(&args.api_address) - .serve(make_service) + .serve(router.into_make_service()) .await .unwrap_or_else(|_| panic!("Failed to bind to address: {}", args.api_address)); } diff --git a/deployer/src/runtime_manager.rs b/deployer/src/runtime_manager.rs index c416556b2..14638d155 100644 --- a/deployer/src/runtime_manager.rs +++ b/deployer/src/runtime_manager.rs @@ -6,7 +6,7 @@ use shuttle_proto::runtime::{ }; use tokio::{process, sync::Mutex}; use tonic::transport::Channel; -use tracing::{info, instrument, trace}; +use tracing::{debug, info, instrument, trace}; use uuid::Uuid; use crate::deployment::deploy_layer; @@ -43,23 +43,27 @@ impl RuntimeManager { pub async fn get_runtime_client( &mut self, - is_next: bool, + legacy_runtime_path: Option, ) -> anyhow::Result> { - if is_next { + if legacy_runtime_path.is_none() { + debug!("Getting shuttle-next runtime client"); + Self::get_runtime_client_helper( &mut self.next, &mut self.next_process, - is_next, + None, self.artifacts_path.clone(), &self.provisioner_address, self.log_sender.clone(), ) .await } else { + debug!("Getting legacy runtime client"); + Self::get_runtime_client_helper( &mut self.legacy, &mut self.legacy_process, - is_next, + legacy_runtime_path, self.artifacts_path.clone(), &self.provisioner_address, self.log_sender.clone(), @@ -101,7 +105,7 @@ impl RuntimeManager { async fn get_runtime_client_helper( runtime_option: &mut Option>, process_option: &mut Option>>, - is_next: bool, + legacy_runtime_path: Option, artifacts_path: PathBuf, provisioner_address: &str, log_sender: crossbeam_channel::Sender, @@ -113,31 +117,50 @@ impl RuntimeManager { trace!("making new client"); let port = portpicker::pick_unused_port().context("failed to find available port")?; + let is_next = legacy_runtime_path.is_none(); let get_runtime_executable = || { - if cfg!(debug_assertions) { - // If we're running deployer natively, install shuttle-runtime using the - // version of runtime from the calling repo. - let path = std::fs::canonicalize(format!("{MANIFEST_DIR}/../runtime")); - - // The path will not be valid if we are in a deployer container, in which - // case we don't try to install and use the one installed in deploy.sh. - if let Ok(path) = path { - std::process::Command::new("cargo") - .arg("install") - .arg("shuttle-runtime") - .arg("--path") - .arg(path) - .output() - .expect("failed to install the local version of shuttle-runtime"); + if let Some(legacy_runtime) = legacy_runtime_path { + debug!( + "Starting legacy runtime at: {}", + legacy_runtime + .clone() + .into_os_string() + .into_string() + .unwrap_or_default() + ); + legacy_runtime + } else { + if cfg!(debug_assertions) { + debug!("Installing shuttle-next runtime in debug mode from local source"); + // If we're running deployer natively, install shuttle-runtime using the + // version of runtime from the calling repo. + let path = std::fs::canonicalize(format!("{MANIFEST_DIR}/../runtime")); + + // The path will not be valid if we are in a deployer container, in which + // case we don't try to install and use the one installed in deploy.sh. + if let Ok(path) = path { + std::process::Command::new("cargo") + .arg("install") + .arg("shuttle-runtime") + .arg("--path") + .arg(path) + .arg("--bin") + .arg("shuttle-next") + .arg("--features") + .arg("next") + .output() + .expect("failed to install the local version of shuttle-runtime"); + } } - } - // If we're in a deployer built with the containerfile, the runtime will have - // been installed in deploy.sh. - home::cargo_home() - .expect("failed to find path to cargo home") - .join("bin/shuttle-runtime") + debug!("Returning path to shuttle-next runtime",); + // If we're in a deployer built with the containerfile, the runtime will have + // been installed in deploy.sh. + home::cargo_home() + .expect("failed to find path to cargo home") + .join("bin/shuttle-next") + } }; let (process, runtime_client) = runtime::start( diff --git a/deployer/tests/deploy_layer/bind-panic/Cargo.toml b/deployer/tests/deploy_layer/bind-panic/Cargo.toml index 2a06bc95d..5c476820e 100644 --- a/deployer/tests/deploy_layer/bind-panic/Cargo.toml +++ b/deployer/tests/deploy_layer/bind-panic/Cargo.toml @@ -3,12 +3,11 @@ name = "bind-panic" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] [dependencies] -shuttle-service = "0.11.0" +shuttle-service = { path = "../../../../service" } +shuttle-runtime = { path = "../../../../runtime" } +tokio = "1.22" diff --git a/deployer/tests/resources/bind-panic/src/lib.rs b/deployer/tests/deploy_layer/bind-panic/src/main.rs similarity index 67% rename from deployer/tests/resources/bind-panic/src/lib.rs rename to deployer/tests/deploy_layer/bind-panic/src/main.rs index 1ecd700bf..b8ccf9c3b 100644 --- a/deployer/tests/resources/bind-panic/src/lib.rs +++ b/deployer/tests/deploy_layer/bind-panic/src/main.rs @@ -4,10 +4,7 @@ struct MyService; #[shuttle_service::async_trait] impl Service for MyService { - async fn bind( - mut self: Box, - _: std::net::SocketAddr, - ) -> Result<(), shuttle_service::Error> { + async fn bind(mut self, _: std::net::SocketAddr) -> Result<(), shuttle_service::Error> { panic!("panic in bind"); } } diff --git a/deployer/tests/deploy_layer/main-panic/Cargo.toml b/deployer/tests/deploy_layer/main-panic/Cargo.toml index 047878b66..157965845 100644 --- a/deployer/tests/deploy_layer/main-panic/Cargo.toml +++ b/deployer/tests/deploy_layer/main-panic/Cargo.toml @@ -3,12 +3,11 @@ name = "main-panic" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] [dependencies] -shuttle-service = "0.11.0" +shuttle-service = { path = "../../../../service" } +shuttle-runtime = { path = "../../../../runtime" } +tokio = "1.22" diff --git a/deployer/tests/resources/main-panic/src/lib.rs b/deployer/tests/deploy_layer/main-panic/src/main.rs similarity index 66% rename from deployer/tests/resources/main-panic/src/lib.rs rename to deployer/tests/deploy_layer/main-panic/src/main.rs index 438c12540..4186a4508 100644 --- a/deployer/tests/resources/main-panic/src/lib.rs +++ b/deployer/tests/deploy_layer/main-panic/src/main.rs @@ -4,10 +4,7 @@ struct MyService; #[shuttle_service::async_trait] impl Service for MyService { - async fn bind( - mut self: Box, - _: std::net::SocketAddr, - ) -> Result<(), shuttle_service::Error> { + async fn bind(mut self, _: std::net::SocketAddr) -> Result<(), shuttle_service::Error> { Ok(()) } } diff --git a/deployer/tests/deploy_layer/self-stop/Cargo.toml b/deployer/tests/deploy_layer/self-stop/Cargo.toml index 1e7b037ae..170cd3670 100644 --- a/deployer/tests/deploy_layer/self-stop/Cargo.toml +++ b/deployer/tests/deploy_layer/self-stop/Cargo.toml @@ -3,12 +3,11 @@ name = "self-stop" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] [dependencies] -shuttle-service = "0.11.0" +shuttle-service = { path = "../../../../service" } +shuttle-runtime = { path = "../../../../runtime" } +tokio = "1.22" diff --git a/deployer/tests/deploy_layer/self-stop/src/lib.rs b/deployer/tests/deploy_layer/self-stop/src/main.rs similarity index 64% rename from deployer/tests/deploy_layer/self-stop/src/lib.rs rename to deployer/tests/deploy_layer/self-stop/src/main.rs index 46558244a..8ba075244 100644 --- a/deployer/tests/deploy_layer/self-stop/src/lib.rs +++ b/deployer/tests/deploy_layer/self-stop/src/main.rs @@ -4,10 +4,7 @@ struct MyService; #[shuttle_service::async_trait] impl Service for MyService { - async fn bind( - mut self: Box, - _: std::net::SocketAddr, - ) -> Result<(), shuttle_service::error::Error> { + async fn bind(mut self, _: std::net::SocketAddr) -> Result<(), shuttle_service::error::Error> { Ok(()) } } diff --git a/deployer/tests/deploy_layer/sleep-async/Cargo.toml b/deployer/tests/deploy_layer/sleep-async/Cargo.toml index d1f5d1a50..3d6e1e4ad 100644 --- a/deployer/tests/deploy_layer/sleep-async/Cargo.toml +++ b/deployer/tests/deploy_layer/sleep-async/Cargo.toml @@ -3,13 +3,11 @@ name = "sleep-async" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] [dependencies] +shuttle-service = { path = "../../../../service" } +shuttle-runtime = { path = "../../../../runtime" } tokio = { version = "1.0", features = ["time"]} -shuttle-service = "0.11.0" diff --git a/deployer/tests/deploy_layer/sleep-async/src/lib.rs b/deployer/tests/deploy_layer/sleep-async/src/main.rs similarity index 76% rename from deployer/tests/deploy_layer/sleep-async/src/lib.rs rename to deployer/tests/deploy_layer/sleep-async/src/main.rs index 542694709..1322552d1 100644 --- a/deployer/tests/deploy_layer/sleep-async/src/lib.rs +++ b/deployer/tests/deploy_layer/sleep-async/src/main.rs @@ -14,10 +14,7 @@ async fn simple() -> Result { #[shuttle_service::async_trait] impl Service for SleepService { - async fn bind( - mut self: Box, - _: std::net::SocketAddr, - ) -> Result<(), shuttle_service::error::Error> { + async fn bind(mut self, _: std::net::SocketAddr) -> Result<(), shuttle_service::error::Error> { let duration = Duration::from_secs(self.duration); sleep(duration).await; diff --git a/deployer/tests/resources/bind-panic/Cargo.toml b/deployer/tests/resources/bind-panic/Cargo.toml index 0a721bd4a..894f3ca8e 100644 --- a/deployer/tests/resources/bind-panic/Cargo.toml +++ b/deployer/tests/resources/bind-panic/Cargo.toml @@ -3,8 +3,6 @@ name = "bind-panic" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -12,3 +10,5 @@ crate-type = ["cdylib"] [dependencies] shuttle-service = { path = "../../../../service" } +shuttle-runtime = { path = "../../../../runtime" } +tokio = "1.22" diff --git a/deployer/tests/deploy_layer/bind-panic/src/lib.rs b/deployer/tests/resources/bind-panic/src/main.rs similarity index 67% rename from deployer/tests/deploy_layer/bind-panic/src/lib.rs rename to deployer/tests/resources/bind-panic/src/main.rs index 1ecd700bf..b8ccf9c3b 100644 --- a/deployer/tests/deploy_layer/bind-panic/src/lib.rs +++ b/deployer/tests/resources/bind-panic/src/main.rs @@ -4,10 +4,7 @@ struct MyService; #[shuttle_service::async_trait] impl Service for MyService { - async fn bind( - mut self: Box, - _: std::net::SocketAddr, - ) -> Result<(), shuttle_service::Error> { + async fn bind(mut self, _: std::net::SocketAddr) -> Result<(), shuttle_service::Error> { panic!("panic in bind"); } } diff --git a/deployer/tests/resources/main-panic/Cargo.toml b/deployer/tests/resources/main-panic/Cargo.toml index da7b5c841..157965845 100644 --- a/deployer/tests/resources/main-panic/Cargo.toml +++ b/deployer/tests/resources/main-panic/Cargo.toml @@ -3,12 +3,11 @@ name = "main-panic" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] [dependencies] shuttle-service = { path = "../../../../service" } +shuttle-runtime = { path = "../../../../runtime" } +tokio = "1.22" diff --git a/deployer/tests/deploy_layer/main-panic/src/lib.rs b/deployer/tests/resources/main-panic/src/main.rs similarity index 66% rename from deployer/tests/deploy_layer/main-panic/src/lib.rs rename to deployer/tests/resources/main-panic/src/main.rs index 438c12540..4186a4508 100644 --- a/deployer/tests/deploy_layer/main-panic/src/lib.rs +++ b/deployer/tests/resources/main-panic/src/main.rs @@ -4,10 +4,7 @@ struct MyService; #[shuttle_service::async_trait] impl Service for MyService { - async fn bind( - mut self: Box, - _: std::net::SocketAddr, - ) -> Result<(), shuttle_service::Error> { + async fn bind(mut self, _: std::net::SocketAddr) -> Result<(), shuttle_service::Error> { Ok(()) } } diff --git a/deployer/tests/resources/sleep-async/Cargo.toml b/deployer/tests/resources/sleep-async/Cargo.toml index c0dc45479..3d6e1e4ad 100644 --- a/deployer/tests/resources/sleep-async/Cargo.toml +++ b/deployer/tests/resources/sleep-async/Cargo.toml @@ -3,13 +3,11 @@ name = "sleep-async" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] [dependencies] -tokio = { version = "1.0", features = ["time"]} shuttle-service = { path = "../../../../service" } +shuttle-runtime = { path = "../../../../runtime" } +tokio = { version = "1.0", features = ["time"]} diff --git a/deployer/tests/resources/sleep-async/src/lib.rs b/deployer/tests/resources/sleep-async/src/main.rs similarity index 76% rename from deployer/tests/resources/sleep-async/src/lib.rs rename to deployer/tests/resources/sleep-async/src/main.rs index 542694709..1322552d1 100644 --- a/deployer/tests/resources/sleep-async/src/lib.rs +++ b/deployer/tests/resources/sleep-async/src/main.rs @@ -14,10 +14,7 @@ async fn simple() -> Result { #[shuttle_service::async_trait] impl Service for SleepService { - async fn bind( - mut self: Box, - _: std::net::SocketAddr, - ) -> Result<(), shuttle_service::error::Error> { + async fn bind(mut self, _: std::net::SocketAddr) -> Result<(), shuttle_service::error::Error> { let duration = Duration::from_secs(self.duration); sleep(duration).await; diff --git a/deployer/tests/resources/tests-fail/Cargo.toml b/deployer/tests/resources/tests-fail/Cargo.toml index 43b6ff357..ae2a76515 100644 --- a/deployer/tests/resources/tests-fail/Cargo.toml +++ b/deployer/tests/resources/tests-fail/Cargo.toml @@ -3,11 +3,10 @@ name = "tests-fail" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] - [workspace] [dependencies] rocket = "0.5.0-rc.2" shuttle-service = { path = "../../../../service", features = ["web-rocket"] } +shuttle-runtime = { path = "../../../../runtime" } +tokio = "1.22" diff --git a/deployer/tests/resources/tests-fail/src/lib.rs b/deployer/tests/resources/tests-fail/src/main.rs similarity index 100% rename from deployer/tests/resources/tests-fail/src/lib.rs rename to deployer/tests/resources/tests-fail/src/main.rs diff --git a/deployer/tests/resources/tests-pass/Cargo.toml b/deployer/tests/resources/tests-pass/Cargo.toml index 43e898a65..ffbd29a9f 100644 --- a/deployer/tests/resources/tests-pass/Cargo.toml +++ b/deployer/tests/resources/tests-pass/Cargo.toml @@ -3,11 +3,10 @@ name = "tests-pass" version = "0.1.0" edition = "2021" -[lib] -crate-type = ["cdylib"] - [workspace] [dependencies] rocket = "0.5.0-rc.2" shuttle-service = { path = "../../../../service", features = ["web-rocket"] } +shuttle-runtime = { path = "../../../../runtime" } +tokio = "1.22" diff --git a/deployer/tests/resources/tests-pass/src/lib.rs b/deployer/tests/resources/tests-pass/src/main.rs similarity index 100% rename from deployer/tests/resources/tests-pass/src/lib.rs rename to deployer/tests/resources/tests-pass/src/main.rs diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index db6ee9d86..60f768767 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -6,9 +6,6 @@ license.workspace = true publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[[bin]] -name = "rocket" - [[bin]] name = "shuttle-next" required-features = ["next"] @@ -37,21 +34,16 @@ wasi-common = { version = "4.0.0", optional = true } wasmtime = { version = "4.0.0", optional = true } wasmtime-wasi = { version = "4.0.0", optional = true } -# For rocket.rs -# TODO: remove -shuttle-secrets = { path = "../resources/secrets" } -rocket = "0.5.0-rc.2" - [dependencies.shuttle-common] workspace = true -features = ["service"] +features = ["service", "backend"] [dependencies.shuttle-proto] workspace = true [dependencies.shuttle-service] workspace = true -features = ["builder", "web-rocket"] # TODO: remove web-rocket +features = ["builder"] [dev-dependencies] crossbeam-channel = "0.5.6" @@ -59,14 +51,4 @@ portpicker = "0.1.1" futures = { version = "0.3.25" } [features] -next = [ - "cap-std", - "futures", - "hyper/server", - "rmp-serde", - "futures", - "wasi-common", - "wasmtime", - "wasmtime-wasi", - "shuttle-common/wasm" -] +next = ["cap-std", "futures", "hyper/server", "rmp-serde", "futures", "wasi-common", "wasmtime", "wasmtime-wasi", "shuttle-common/wasm"] diff --git a/runtime/src/bin/rocket.rs b/runtime/src/bin/rocket.rs deleted file mode 100644 index db3836826..000000000 --- a/runtime/src/bin/rocket.rs +++ /dev/null @@ -1,51 +0,0 @@ -// The few line below is what we should now codegen for legacy -#[tokio::main] -async fn main() { - shuttle_runtime::start(loader).await; -} - -async fn loader( - mut factory: shuttle_runtime::ProvisionerFactory, - _logger: shuttle_runtime::Logger, -) -> shuttle_service::ShuttleRocket { - use shuttle_service::ResourceBuilder; - - let secrets = shuttle_secrets::Secrets::new().build(&mut factory).await?; - - rocket(secrets).await -} - -// Everything below this is the usual code a user will write -use anyhow::anyhow; -use rocket::response::status::BadRequest; -use rocket::State; -use shuttle_secrets::SecretStore; - -#[rocket::get("/secret")] -async fn secret(state: &State) -> Result> { - Ok(state.secret.clone()) -} - -struct MyState { - secret: String, -} - -// #[shuttle_service::main] -pub async fn rocket( - // #[shuttle_secrets::Secrets] secret_store: SecretStore, - secret_store: SecretStore, -) -> shuttle_service::ShuttleRocket { - // get secret defined in `Secrets.toml` file. - let secret = if let Some(secret) = secret_store.get("MY_API_KEY") { - secret - } else { - return Err(anyhow!("secret was not found").into()); - }; - - let state = MyState { secret }; - let rocket = rocket::build() - .mount("/", rocket::routes![secret]) - .manage(state); - - Ok(rocket) -} diff --git a/runtime/src/legacy/mod.rs b/runtime/src/legacy/mod.rs index fa8bce121..5d430bbc1 100644 --- a/runtime/src/legacy/mod.rs +++ b/runtime/src/legacy/mod.rs @@ -4,7 +4,7 @@ use std::{ net::{Ipv4Addr, SocketAddr}, ops::DerefMut, str::FromStr, - sync::Mutex, + sync::{Arc, Mutex}, time::Duration, }; @@ -14,7 +14,7 @@ use clap::Parser; use core::future::Future; use shuttle_common::{ backends::{auth::ClaimLayer, tracing::InjectPropagationLayer}, - storage_manager::{StorageManager, WorkingDirStorageManager}, + storage_manager::{ArtifactsStorageManager, StorageManager, WorkingDirStorageManager}, LogItem, }; use shuttle_proto::{ @@ -47,9 +47,7 @@ use self::args::Args; mod args; -pub async fn start( - loader: impl Loader> + Send + 'static, -) { +pub async fn start(loader: impl Loader + Send + 'static) { let args = Args::parse(); let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), args.port); @@ -57,11 +55,24 @@ pub async fn start( let mut server_builder = Server::builder().http2_keepalive_interval(Some(Duration::from_secs(60))); + // We wrap the StorageManager trait object in an Arc rather than a Box, since we need + // to clone it in the `ProvisionerFactory::new` call in the legacy runtime `load` method. + // We might be able to optimize this by implementing clone for a Box + // or by using static dispatch instead. + let storage_manager: Arc = match args.storage_manager_type { + args::StorageManagerType::Artifacts => { + Arc::new(ArtifactsStorageManager::new(args.storage_manager_path)) + } + args::StorageManagerType::WorkingDir => { + Arc::new(WorkingDirStorageManager::new(args.storage_manager_path)) + } + }; + let router = { let legacy = Legacy::new( provisioner_address, loader, - WorkingDirStorageManager::new(args.storage_manager_path), + storage_manager, Environment::Local, ); @@ -72,24 +83,24 @@ pub async fn start( router.serve(addr).await.unwrap(); } -pub struct Legacy { +pub struct Legacy { // Mutexes are for interior mutability logs_rx: Mutex>>, logs_tx: UnboundedSender, stopped_tx: Sender<(StopReason, String)>, provisioner_address: Endpoint, kill_tx: Mutex>>, - storage_manager: M, + storage_manager: Arc, loader: Mutex>, service: Mutex>, env: Environment, } -impl Legacy { +impl Legacy { pub fn new( provisioner_address: Endpoint, loader: L, - storage_manager: M, + storage_manager: Arc, env: Environment, ) -> Self { let (tx, rx) = mpsc::unbounded_channel(); @@ -143,10 +154,9 @@ where } #[async_trait] -impl Runtime for Legacy +impl Runtime for Legacy where - M: StorageManager + Send + Sync + 'static, - L: Loader, Service = S> + Send + 'static, + L: Loader + Send + 'static, S: Service + Send + 'static, { async fn load(&self, request: Request) -> Result, Status> { @@ -155,7 +165,7 @@ where secrets, service_name, } = request.into_inner(); - trace!(path, "loading"); + trace!(path, "loading legacy project"); let secrets = BTreeMap::from_iter(secrets.into_iter()); diff --git a/runtime/src/next/mod.rs b/runtime/src/next/mod.rs index 027d16c25..6c43948d5 100644 --- a/runtime/src/next/mod.rs +++ b/runtime/src/next/mod.rs @@ -79,7 +79,7 @@ impl Runtime for AxumWasm { request: tonic::Request, ) -> Result, Status> { let wasm_path = request.into_inner().path; - trace!(wasm_path, "loading"); + trace!(wasm_path, "loading shuttle-next project"); let router = RouterBuilder::new() .map_err(|err| Status::from_error(err.into()))? diff --git a/runtime/src/provisioner_factory.rs b/runtime/src/provisioner_factory.rs index 67a6f1957..8193af78e 100644 --- a/runtime/src/provisioner_factory.rs +++ b/runtime/src/provisioner_factory.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, path::PathBuf}; +use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; use async_trait::async_trait; use shuttle_common::{ @@ -16,29 +16,23 @@ use tracing::{debug, info, trace}; use uuid::Uuid; /// A factory (service locator) which goes through the provisioner crate -pub struct ProvisionerFactory -where - S: StorageManager, -{ +pub struct ProvisionerFactory { service_name: ServiceName, deployment_id: Uuid, - storage_manager: S, + storage_manager: Arc, provisioner_client: ProvisionerClient>>, info: Option, secrets: BTreeMap, env: Environment, } -impl ProvisionerFactory -where - S: StorageManager, -{ +impl ProvisionerFactory { pub(crate) fn new( provisioner_client: ProvisionerClient>>, service_name: ServiceName, deployment_id: Uuid, secrets: BTreeMap, - storage_manager: S, + storage_manager: Arc, env: Environment, ) -> Self { Self { @@ -54,10 +48,7 @@ where } #[async_trait] -impl Factory for ProvisionerFactory -where - S: StorageManager + Sync + Send, -{ +impl Factory for ProvisionerFactory { async fn get_db_connection_string( &mut self, db_type: database::Type,