feat: static file support for a single folder (#501)

* feat: static-folder resource * refactor: make local client work * refactor: make local deployer work * feat: storage_manager * refactor: comments * refactor: remove unwraps and expects * refactor: update tests * refactor: clippy suggestion * refactor: update version * refactor: update readme * refactor: update comment * refactor: change public to static * ci: add static-folder * refactor: code fixes * refactor: update tests * refactor: update description Co-authored-by: Damien <[email protected]> Co-authored-by: Damien <[email protected]>
shuttle-hq · Dec 1, 2022 · 6c3025b · 6c3025b
1 parent c2fa52c
commit 6c3025b
Show file tree

Hide file tree

Showing 19 changed files with 376 additions and 84 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -89,6 +89,7 @@ commands:
             shuttle-persist = { path = "$PWD/resources/persist" }
             shuttle-shared-db = { path = "$PWD/resources/shared-db" }
             shuttle-secrets = { path = "$PWD/resources/secrets" }
+            shuttle-static-folder = { path = "$PWD/resources/static-folder" }
             EOF
   install-rust:
     steps:
@@ -260,6 +261,7 @@ workflows:
                 - resources/persist
                 - resources/secrets
                 - resources/shared-db
+                - resources/static-folder
       - service-test:
           requires:
             - workspace-clippy

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -49,6 +49,7 @@ shuttle-aws-rds = { path = "[base]/shuttle/resources/aws-rds" }
 shuttle-persist = { path = "[base]/shuttle/resources/persist" }
 shuttle-shared-db = { path = "[base]/shuttle/resources/shared-db" }
 shuttle-secrets = { path = "[base]/shuttle/resources/secrets" }
+shuttle-static-folder = { path = "[base]/shuttle/resources/static-folder" }
 ```
 
 Prime gateway database with an admin user:

diff --git a/Cargo.toml b/Cargo.toml
@@ -16,5 +16,6 @@ exclude = [
   "resources/aws-rds",
   "resources/persist",
   "resources/secrets",
-  "resources/shared-db"
+  "resources/shared-db",
+  "resources/static-folder"
 ]
diff --git a/Makefile b/Makefile
@@ -188,6 +188,7 @@ publish: publish-resources publish-cargo-shuttle
 publish-resources: publish-resources/aws-rds \
 	publish-resources/persist \
 	publish-resources/shared-db
+	publish-resources/static-folder
 
 publish-cargo-shuttle: publish-resources/secrets
 	cd cargo-shuttle; cargo publish

diff --git a/cargo-shuttle/src/factory.rs b/cargo-shuttle/src/factory.rs
@@ -23,6 +23,7 @@ use shuttle_service::{database::Type, error::CustomError, Factory, ServiceName};
 use std::{
     collections::{BTreeMap, HashMap},
     io::stdout,
+    path::PathBuf,
     time::Duration,
 };
 use tokio::time::sleep;
@@ -32,14 +33,20 @@ pub struct LocalFactory {
     docker: Docker,
     service_name: ServiceName,
     secrets: BTreeMap<String, String>,
+    working_directory: PathBuf,
 }
 
 impl LocalFactory {
-    pub fn new(service_name: ServiceName, secrets: BTreeMap<String, String>) -> Result<Self> {
+    pub fn new(
+        service_name: ServiceName,
+        secrets: BTreeMap<String, String>,
+        working_directory: PathBuf,
+    ) -> Result<Self> {
         Ok(Self {
             docker: Docker::connect_with_local_defaults()?,
             service_name,
             secrets,
+            working_directory,
         })
     }
 }
@@ -176,6 +183,14 @@ impl Factory for LocalFactory {
     fn get_service_name(&self) -> ServiceName {
         self.service_name.clone()
     }
+
+    fn get_build_path(&self) -> Result<PathBuf, shuttle_service::Error> {
+        Ok(self.working_directory.clone())
+    }
+
+    fn get_storage_path(&self) -> Result<PathBuf, shuttle_service::Error> {
+        Ok(self.working_directory.clone())
+    }
 }
 
 impl LocalFactory {

diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs
@@ -309,7 +309,11 @@ impl Shuttle {
 
         let loader = Loader::from_so_file(so_path)?;
 
-        let mut factory = LocalFactory::new(self.ctx.project_name().clone(), secrets)?;
+        let mut factory = LocalFactory::new(
+            self.ctx.project_name().clone(),
+            secrets,
+            working_directory.to_path_buf(),
+        )?;
         let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), run_args.port);
 
         trace!("loading project");

diff --git a/deployer/prepare.sh b/deployer/prepare.sh
@@ -12,7 +12,8 @@ shuttle-service = { path = "/usr/src/shuttle/service" }
 shuttle-aws-rds = { path = "/usr/src/shuttle/resources/aws-rds" }
 shuttle-persist = { path = "/usr/src/shuttle/resources/persist" }
 shuttle-shared-db = { path = "/usr/src/shuttle/resources/shared-db" }
-shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" }' > $CARGO_HOME/config.toml
+shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" }
+shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' > $CARGO_HOME/config.toml
 
 # Prefetch crates.io index
 cd /usr/src/shuttle/service

diff --git a/deployer/src/deployment/deploy_layer.rs b/deployer/src/deployment/deploy_layer.rs
@@ -362,8 +362,9 @@ mod tests {
 
     use crate::{
         deployment::{
-            deploy_layer::LogType, provisioner_factory, runtime_logger, ActiveDeploymentsGetter,
-            Built, DeploymentManager, Queued,
+            deploy_layer::LogType, provisioner_factory, runtime_logger,
+            storage_manager::StorageManager, ActiveDeploymentsGetter, Built, DeploymentManager,
+            Queued,
         },
         persistence::{SecretRecorder, State},
     };
@@ -460,6 +461,8 @@ mod tests {
             &self,
             _project_name: shuttle_common::project::ProjectName,
             _service_id: Uuid,
+            _deployment_id: Uuid,
+            _storage_manager: StorageManager,
         ) -> Result<Self::Output, Self::Error> {
             Ok(StubProvisionerFactory)
         }
@@ -485,6 +488,14 @@ mod tests {
         fn get_service_name(&self) -> shuttle_service::ServiceName {
             panic!("did not expect any deploy_layer test to get the service name")
         }
+
+        fn get_build_path(&self) -> Result<PathBuf, shuttle_service::Error> {
+            panic!("did not expect any deploy_layer test to get the build path")
+        }
+
+        fn get_storage_path(&self) -> Result<PathBuf, shuttle_service::Error> {
+            panic!("did not expect any deploy_layer test to get the storage path")
+        }
     }
 
     struct StubRuntimeLoggerFactory;

diff --git a/deployer/src/deployment/mod.rs b/deployer/src/deployment/mod.rs
@@ -3,6 +3,7 @@ pub mod provisioner_factory;
 mod queue;
 mod run;
 pub mod runtime_logger;
+mod storage_manager;
 
 use std::path::PathBuf;
 
@@ -15,7 +16,7 @@ use crate::persistence::{SecretRecorder, State};
 use tokio::sync::{broadcast, mpsc};
 use uuid::Uuid;
 
-use self::deploy_layer::LogRecorder;
+use self::{deploy_layer::LogRecorder, storage_manager::StorageManager};
 
 const QUEUE_BUFFER_SIZE: usize = 100;
 const RUN_BUFFER_SIZE: usize = 100;
@@ -48,7 +49,7 @@ impl DeploymentManager {
                 build_log_recorder,
                 secret_recorder,
                 active_deployment_getter,
-                artifacts_path,
+                StorageManager::new(artifacts_path),
             ),
             kill_send,
         }
@@ -109,7 +110,7 @@ impl Pipeline {
         build_log_recorder: impl LogRecorder,
         secret_recorder: impl SecretRecorder,
         active_deployment_getter: impl ActiveDeploymentsGetter,
-        artifacts_path: PathBuf,
+        storage_manager: StorageManager,
     ) -> Pipeline {
         let (queue_send, queue_recv) = mpsc::channel(QUEUE_BUFFER_SIZE);
         let (run_send, run_recv) = mpsc::channel(RUN_BUFFER_SIZE);
@@ -121,15 +122,15 @@ impl Pipeline {
             run_send_clone,
             build_log_recorder,
             secret_recorder,
-            artifacts_path.clone(),
+            storage_manager.clone(),
         ));
         tokio::spawn(run::task(
             run_recv,
             kill_send,
             abstract_factory,
             runtime_logger_factory,
             active_deployment_getter,
-            artifacts_path,
+            storage_manager,
         ));
 
         Pipeline {

diff --git a/deployer/src/deployment/provisioner_factory.rs b/deployer/src/deployment/provisioner_factory.rs
@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, path::PathBuf};
 
 use async_trait::async_trait;
 use shuttle_common::{database, DatabaseReadyInfo};
@@ -16,6 +16,8 @@ use uuid::Uuid;
 
 use crate::persistence::{Resource, ResourceRecorder, ResourceType, SecretGetter};
 
+use super::storage_manager::StorageManager;
+
 /// Trait to make it easy to get a factory (service locator) for each service being started
 #[async_trait]
 pub trait AbstractFactory: Send + Sync + 'static {
@@ -27,6 +29,8 @@ pub trait AbstractFactory: Send + Sync + 'static {
         &self,
         service_name: ServiceName,
         service_id: Uuid,
+        deployment_id: Uuid,
+        storage_manager: StorageManager,
     ) -> Result<Self::Output, Self::Error>;
 }
 
@@ -47,13 +51,17 @@ impl<R: ResourceRecorder, S: SecretGetter> AbstractFactory for AbstractProvision
         &self,
         service_name: ServiceName,
         service_id: Uuid,
+        deployment_id: Uuid,
+        storage_manager: StorageManager,
     ) -> Result<Self::Output, Self::Error> {
         let provisioner_client = ProvisionerClient::connect(self.provisioner_uri.clone()).await?;
 
         Ok(ProvisionerFactory::new(
             provisioner_client,
             service_name,
             service_id,
+            deployment_id,
+            storage_manager,
             self.resource_recorder.clone(),
             self.secret_getter.clone(),
         ))
@@ -80,6 +88,8 @@ pub enum ProvisionerError {
 pub struct ProvisionerFactory<R: ResourceRecorder, S: SecretGetter> {
     service_name: ServiceName,
     service_id: Uuid,
+    deployment_id: Uuid,
+    storage_manager: StorageManager,
     provisioner_client: ProvisionerClient<Channel>,
     info: Option<DatabaseReadyInfo>,
     resource_recorder: R,
@@ -92,13 +102,17 @@ impl<R: ResourceRecorder, S: SecretGetter> ProvisionerFactory<R, S> {
         provisioner_client: ProvisionerClient<Channel>,
         service_name: ServiceName,
         service_id: Uuid,
+        deployment_id: Uuid,
+        storage_manager: StorageManager,
         resource_recorder: R,
         secret_getter: S,
     ) -> Self {
         Self {
             provisioner_client,
             service_name,
             service_id,
+            deployment_id,
+            storage_manager,
             info: None,
             resource_recorder,
             secret_getter,
@@ -179,4 +193,16 @@ impl<R: ResourceRecorder, S: SecretGetter> Factory for ProvisionerFactory<R, S>
     fn get_service_name(&self) -> ServiceName {
         self.service_name.clone()
     }
+
+    fn get_build_path(&self) -> Result<PathBuf, shuttle_service::Error> {
+        self.storage_manager
+            .service_build_path(self.service_name.as_str())
+            .map_err(Into::into)
+    }
+
+    fn get_storage_path(&self) -> Result<PathBuf, shuttle_service::Error> {
+        self.storage_manager
+            .deployment_storage_path(self.service_name.as_str(), &self.deployment_id)
+            .map_err(Into::into)
+    }
 }