feat: simple local run (#186)

* feat: simple startup

* refactor: extract factory

* refactor: extract log printing

* feat: add building support

* refactor: clippy suggestions

* tests: for crate builder

* refactor: split client into binary and lib

* tests: move binary tests to library tests

* tests: client run test

* refactor: try async channel

* bug: fix 'invalid memory reference' for tokio channel

* refactor: make ports unique for each test

* bug: runtime logs in deployment

* refactor: clippy suggestions

* tests: config docs

* feat: debug info for local-run

* docs: local-run

* refactor: make unsupported DBs clearer

api/src/build.rs

@@ -3,11 +3,9 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 
 use anyhow::{anyhow, Context, Result};
-use cargo::core::compiler::CompileMode;
-use cargo::core::Workspace;
-use cargo::ops::CompileOptions;
 use rocket::tokio;
 use rocket::tokio::io::AsyncWriteExt;
+use shuttle_service::loader::build_crate;
 use uuid::Uuid;
 
 #[cfg(debug_assertions)]
@@ -190,31 +188,3 @@ fn extract_tarball(crate_path: &Path, project_path: &Path) -> Result<()> {
         Ok(())
     }
 }
-
-/// Given a project directory path, builds the crate
-fn build_crate(project_path: &Path, buf: Box<dyn std::io::Write>) -> Result<PathBuf> {
-    let mut shell = cargo::core::Shell::from_write(buf);
-    shell.set_verbosity(cargo::core::Verbosity::Normal);
-
-    let cwd = std::env::current_dir()
-        .with_context(|| "couldn't get the current directory of the process")?;
-    let homedir = cargo::util::homedir(&cwd).ok_or_else(|| {
-        anyhow!(
-            "Cargo couldn't find your home directory. \
-                 This probably means that $HOME was not set."
-        )
-    })?;
-
-    let config = cargo::Config::new(shell, cwd, homedir);
-    let manifest_path = project_path.join("Cargo.toml");
-
-    let ws = Workspace::new(&manifest_path, &config)?;
-    let opts = CompileOptions::new(&config, CompileMode::Build)?;
-    let compilation = cargo::ops::compile(&ws, &opts)?;
-
-    if compilation.cdylibs.is_empty() {
-        return Err(anyhow!("a cdylib was not created"));
-    }
-
-    Ok(compilation.cdylibs[0].path.clone())
-}

api/src/deployment.rs

@@ -4,7 +4,6 @@ use std::fs::DirEntry;
 use std::io::Write;
 use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4, TcpListener};
 use std::path::{Path, PathBuf};
-use std::sync::mpsc::SyncSender;
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context as AnyhowContext};
@@ -20,6 +19,7 @@ use shuttle_common::{
 use shuttle_service::loader::Loader;
 use shuttle_service::logger::Log;
 use shuttle_service::ServeHandle;
+use tokio::sync::mpsc::UnboundedSender;
 use tokio::sync::{mpsc, RwLock};
 
 use crate::build::Build;
@@ -109,7 +109,7 @@ impl Deployment {
         &self,
         context: &Context,
         db_context: &database::Context,
-        run_logs_tx: SyncSender<Log>,
+        run_logs_tx: UnboundedSender<Log>,
     ) {
         {
             trace!("waiting to get write on the state");
@@ -301,7 +301,7 @@ impl JobQueue {
     async fn new(
         context: Context,
         db_context: database::Context,
-        run_logs_tx: SyncSender<Log>,
+        run_logs_tx: UnboundedSender<Log>,
     ) -> Self {
         let (send, mut recv) = mpsc::channel::<Arc<Deployment>>(JOB_QUEUE_SIZE);
 
@@ -347,7 +347,7 @@ pub(crate) struct Context {
 impl DeploymentSystem {
     pub(crate) async fn new(build_system: Box<dyn BuildSystem>, fqdn: String) -> Self {
         let router: Arc<Router> = Default::default();
-        let (tx, rx) = std::sync::mpsc::sync_channel::<Log>(64);
+        let (tx, mut rx) = mpsc::unbounded_channel::<Log>();
 
         let deployments = Arc::new(RwLock::new(
             Self::initialise_from_fs(&build_system.fs_root(), &fqdn).await,
@@ -356,16 +356,11 @@ impl DeploymentSystem {
         let deployments_log = deployments.clone();
 
         tokio::spawn(async move {
-            loop {
-                let res = rx.recv();
+            while let Some(log) = rx.recv().await {
+                let mut deployments_log = deployments_log.write().await;
 
-                if let Ok(log) = res {
-                    let mut deployments_log = deployments_log.write().await;
-                    if let Some(deployment) = deployments_log.get_mut(&log.deployment_id) {
-                        deployment.add_runtime_log(log.datetime, log.item).await;
-                    }
-                } else {
-                    break;
+                if let Some(deployment) = deployments_log.get_mut(&log.deployment_id) {
+                    deployment.add_runtime_log(log.datetime, log.item).await;
                 }
             }
         });

cargo-shuttle/Cargo.toml

@@ -7,10 +7,12 @@ description = "A cargo command for the shuttle platform (https://www.shuttle.rs/
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = "1.0.53"
+anyhow = "1.0.54"
+async-trait = "0.1.52"
 chrono = "0.4"
 colored = "2.0"
 dirs = "4.0.0"
+env_logger = "0.9.0"
 log = "0.4"
 serde_json = "1.0.79"
 serde = { version = "1.0.136", features = ["derive"] }
@@ -24,9 +26,10 @@ toml = "0.5.8"
 structopt = "0.3.26"
 cargo_metadata = "0.14.2"
 webbrowser = "0.6"
+uuid = { version = "0.8.2", features = ["v4"] }
 
 shuttle-common = { version = "0.3.0", path = "../common" }
+shuttle-service = { version = "0.3.0", path = "../service", features = [ "loader" ] }
 
 [dev-dependencies]
-assert_cmd = "2.0.4"
-predicates = "2.1.1"
+portpicker = "0.1.1"

cargo-shuttle/src/args.rs

@@ -31,7 +31,7 @@ pub struct Args {
 }
 
 // Common args for subcommands that deal with projects.
-#[derive(StructOpt)]
+#[derive(StructOpt, Debug)]
 pub struct ProjectArgs {
     #[structopt(
         global = true,
@@ -65,6 +65,8 @@ pub enum Command {
     Auth(AuthArgs),
     #[structopt(about = "login to the shuttle platform")]
     Login(LoginArgs),
+    #[structopt(about = "run a shuttle project locally")]
+    Run(RunArgs),
 }
 
 #[derive(StructOpt)]
@@ -84,3 +86,9 @@ pub struct DeployArgs {
     #[structopt(long, about = "allow dirty working directories to be packaged")]
     pub allow_dirty: bool,
 }
+
+#[derive(StructOpt, Debug)]
+pub struct RunArgs {
+    #[structopt(long, about = "port to start service on", default_value = "8000")]
+    pub port: u16,
+}

cargo-shuttle/src/client.rs

@@ -4,9 +4,6 @@ use std::io::Read;
 use std::time::Duration;
 
 use anyhow::{anyhow, Context, Result};
-use chrono::{DateTime, Local};
-use colored::{ColoredString, Colorize};
-use log::Level;
 use reqwest::{Response, StatusCode};
 use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
 use reqwest_retry::policies::ExponentialBackoff;
@@ -15,6 +12,8 @@ use shuttle_common::project::ProjectName;
 use shuttle_common::{ApiKey, ApiUrl, DeploymentMeta, DeploymentStateMeta, SHUTTLE_PROJECT_HEADER};
 use tokio::time::sleep;
 
+use crate::print;
+
 pub(crate) async fn auth(api_url: ApiUrl, username: String) -> Result<ApiKey> {
     let client = get_retry_client();
     let mut api_url = api_url;
@@ -75,32 +74,13 @@ pub(crate) async fn logs(api_url: ApiUrl, api_key: &ApiKey, project: &ProjectNam
 
     let deployment_meta = get_deployment_meta(api_url, api_key, project, &client).await?;
 
-    for (datetime, log) in deployment_meta.runtime_logs {
-        let datetime: DateTime<Local> = DateTime::from(datetime);
-        println!(
-            "{}{} {:<5} {}{} {}",
-            "[".bright_black(),
-            datetime.format("%Y-%m-%dT%H:%M:%SZ"),
-            get_colored_level(&log.level),
-            log.target,
-            "]".bright_black(),
-            log.body
-        );
+    for (datetime, log_item) in deployment_meta.runtime_logs {
+        print::log(datetime, log_item);
     }
 
     Ok(())
 }
 
-fn get_colored_level(level: &Level) -> ColoredString {
-    match level {
-        Level::Trace => level.to_string().bright_black(),
-        Level::Debug => level.to_string().blue(),
-        Level::Info => level.to_string().green(),
-        Level::Warn => level.to_string().yellow(),
-        Level::Error => level.to_string().red(),
-    }
-}
-
 async fn get_deployment_meta(
     api_url: ApiUrl,
     api_key: &ApiKey,