feat: refactor deployer to run locally without auth (#810)

* feat: refactor deployer to run locally without auth

* docs: update contributing deployer guide

* refactor: workspace dep

* fix: bump regex to 1.8.1 to fix 1.8.0 bug

* fix: provisioner port for local deployer

* refactor: renaming

* feat: refactor to request token from auth

* refactor: remove claims feature from deployer common

* refactor: use key/cookie from original request

* refactor: implement scopebuilder

* fix: clippy

* refactor: address review

* refactor: auth container command

* refactor: cleanup builder in start fn

* docs: local arg docs

CONTRIBUTING.md

@@ -148,60 +148,54 @@ cargo run --manifest-path ../../../Cargo.toml --bin cargo-shuttle -- logs
 The steps outlined above starts all the services used by shuttle locally (ie. both `gateway` and `deployer`). However, sometimes you will want to quickly test changes to `deployer` only. To do this replace `make up` with the following:
 
 ```bash
-# first generate the local docker-compose file
+# if you didn't do this already, make the images
+USE_PANAMAX=disable make images
+
+# then generate the local docker-compose file
 make docker-compose.rendered.yml
 
 # then run it
 docker compose -f docker-compose.rendered.yml up provisioner
 ```
 
-This starts the provisioner and the auth service, while preventing `gateway` from starting up. Next up we need to
-insert an admin user into the `auth` state using the ID of the `auth` container and the auth CLI `init` command:
+This starts the provisioner and the auth service, while preventing `gateway` from starting up.
+Next up we need to insert an admin user into the `auth` state using the ID of the `auth`
+container and the auth CLI `init` command:
 
 ```bash
-AUTH_CONTAINER_ID=$(docker ps -aqf "name=shuttle-auth") \
+AUTH_CONTAINER_ID=$(docker ps -qf "name=auth") \
     docker exec $AUTH_CONTAINER_ID ./usr/local/bin/service \
     --state=/var/lib/shuttle-auth \
     init --name admin --key test-key
 ```
+> Note: if you have done this already for this container you will get a "UNIQUE constraint failed"
+> error, you can ignore this.
 
-Before we can run commands against a local deployer, we need to get a valid JWT and set it in our
-`.config/shuttle/config.toml` as our `api_key`. By running the following curl command, we will request
-that our api-key in the `Authorization` header be converted to a JWT, which will be returned in the response:
+We need to make sure we're logged in with the same key we inserted for the admin user in the
+previous step:
 
 ```bash
-curl -H "Authorization: Bearer test-key" localhost:8008/auth/key
+cargo shuttle login --api-key test-key
 ```
 
-Now copy the `token` value (just the value, not the key) from the curl response, and write it to your shuttle
-config (which will be a file named `config.toml` in a directory named `shuttle` in one of 
-[these places](https://docs.rs/dirs/latest/dirs/fn.config_dir.html) depending on your OS).
-
-```bash
-# replace <jwt> with the token from the previous command
-echo "api_key = '<jwt>'" > ~/.config/shuttle/config.toml
-```
+We're now ready to start a local run of the deployer:
 
-> Note: The JWT will expire in 15 minutes, at which point you need to run the commands again.
-> If you have [`jq`](https://github.com/stedolan/jq/wiki/Installation) installed you can combine
-> the two above commands into the following:
 ```bash
-curl -s -H "Authorization: Bearer test-key" localhost:8008/auth/key \
-    | jq -r '.token' \
-    | read token; echo "api_key='$token'" > ~/.config/shuttle/config.toml
+cargo run -p shuttle-deployer -- --provisioner-address http://localhost:5000 --auth-uri http://localhost:8008 --proxy-fqdn local.rs --admin-secret test-key --local --project <project_name>
 ```
 
-Finally we need to comment out the admin layer in the deployer handlers. So in `deployer/handlers/mod.rs`,
-in the `make_router` function comment out this line: `.layer(AdminSecretLayer::new(admin_secret))`.
+The `<project_name>` needs to match the name of the project that will be deployed to this deployer. This is the `Cargo.toml` or `Shuttle.toml` name for the project.
 
-And that's it, we're ready to start our deployer!
+Now that your local deployer is running, you can run commands against using the cargo-shuttle CLI.
+To do that you should navigate into an example, it needs to have the same project name as the
+one you submitted when starting the deployer above. Then you can use the CLI like you normally
+would:
 
 ```bash
-cargo run -p shuttle-deployer -- --provisioner-address http://localhost:8000 --proxy-fqdn local.rs --admin-secret test-key --project <project_name>
+# the manifest path is the path to the root shuttle manifest from the example directory
+cargo run --bin cargo-shuttle --manifest-path="../../../Cargo.toml" -- deploy
 ```
 
-The `--admin-secret` can safely be changed to your api-key to make testing easier. While `<project_name>` needs to match the name of the project that will be deployed to this deployer. This is the `Cargo.toml` or `Shuttle.toml` name for the project.
-
 ### Using Podman instead of Docker
 
 If you want to use Podman instead of Docker, you can configure the build process with environment variables.

Cargo.toml

@@ -16,11 +16,7 @@ members = [
 exclude = [
   "e2e",
   "examples",
-  "resources/aws-rds",
-  "resources/persist",
-  "resources/secrets",
-  "resources/shared-db",
-  "resources/static-folder",
+  "resources",
   "services",
 ]
 

auth/src/user.rs

@@ -9,7 +9,7 @@ use axum::{
 };
 use rand::distributions::{Alphanumeric, DistString};
 use serde::{Deserialize, Deserializer, Serialize};
-use shuttle_common::claims::Scope;
+use shuttle_common::claims::{Scope, ScopeBuilder};
 use sqlx::{query, Row, SqlitePool};
 use tracing::{trace, Span};
 
@@ -185,33 +185,13 @@ pub enum AccountTier {
 
 impl From<AccountTier> for Vec<Scope> {
     fn from(tier: AccountTier) -> Self {
-        let mut base = vec![
-            Scope::Deployment,
-            Scope::DeploymentPush,
-            Scope::Logs,
-            Scope::Service,
-            Scope::ServiceCreate,
-            Scope::Project,
-            Scope::ProjectCreate,
-            Scope::Resources,
-            Scope::ResourcesWrite,
-            Scope::Secret,
-            Scope::SecretWrite,
-        ];
+        let mut builder = ScopeBuilder::new();
 
         if tier == AccountTier::Admin {
-            base.append(&mut vec![
-                Scope::User,
-                Scope::UserCreate,
-                Scope::AcmeCreate,
-                Scope::CustomDomainCreate,
-                Scope::CustomDomainCertificateRenew,
-                Scope::GatewayCertificateRenew,
-                Scope::Admin,
-            ]);
+            builder = builder.with_admin()
         }
 
-        base
+        builder.build()
     }
 }
 

common/Cargo.toml

@@ -94,4 +94,4 @@ ring = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 tower = { workspace = true, features = ["util"] }
 tracing-fluent-assertions = "0.3.0"
-tracing-subscriber = { version = "0.3", default-features = false }
+tracing-subscriber = { workspace = true }

common/src/claims.rs

@@ -88,6 +88,55 @@ pub enum Scope {
     Admin,
 }
 
+pub struct ScopeBuilder(Vec<Scope>);
+
+impl ScopeBuilder {
+    /// Create a builder with the standard scopes for new users.
+    pub fn new() -> Self {
+        Self(vec![
+            Scope::Deployment,
+            Scope::DeploymentPush,
+            Scope::Logs,
+            Scope::Service,
+            Scope::ServiceCreate,
+            Scope::Project,
+            Scope::ProjectCreate,
+            Scope::Resources,
+            Scope::ResourcesWrite,
+            Scope::Secret,
+            Scope::SecretWrite,
+        ])
+    }
+
+    /// Extend the current scopes with admin scopes.
+    pub fn with_admin(mut self) -> Self {
+        self.0.extend(vec![
+            Scope::Deployment,
+            Scope::DeploymentPush,
+            Scope::Logs,
+            Scope::Service,
+            Scope::ServiceCreate,
+            Scope::Project,
+            Scope::ProjectCreate,
+            Scope::Resources,
+            Scope::ResourcesWrite,
+            Scope::Secret,
+            Scope::SecretWrite,
+        ]);
+        self
+    }
+
+    pub fn build(self) -> Vec<Scope> {
+        self.0
+    }
+}
+
+impl Default for ScopeBuilder {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
 pub struct Claim {
     /// Expiration time (as UTC timestamp).

deployer/src/args.rs

@@ -50,4 +50,8 @@ pub struct Args {
     /// Uri to folder to store all artifacts
     #[clap(long, default_value = "/tmp")]
     pub artifacts_path: PathBuf,
+
+    /// Add an auth layer to deployer for local development
+    #[arg(long)]
+    pub local: bool,
 }

deployer/src/handlers/local.rs

@@ -0,0 +1,86 @@
+use std::net::Ipv4Addr;
+
+use axum::{
+    headers::{authorization::Bearer, Authorization, Cookie, Header, HeaderMapExt},
+    http::Request,
+    middleware::Next,
+    response::Response,
+    Extension,
+};
+use hyper::{
+    client::{connect::dns::GaiResolver, HttpConnector},
+    Body, Client, StatusCode, Uri,
+};
+use hyper_reverse_proxy::ReverseProxy;
+use once_cell::sync::Lazy;
+use serde_json::Value;
+use tracing::error;
+
+static PROXY_CLIENT: Lazy<ReverseProxy<HttpConnector<GaiResolver>>> =
+    Lazy::new(|| ReverseProxy::new(Client::new()));
+
+/// This middleware proxies a request to the auth service to get a JWT, which we need to access
+/// the deployer endpoints, and we'll also need it in the claim layer of the provisioner and runtime
+/// clients.
+///
+/// Follow the steps in https://github.com/shuttle-hq/shuttle/blob/main/CONTRIBUTING.md#testing-deployer-only
+/// to learn how to insert an admin user in the auth state.
+///
+/// WARNING: do not set this layer in production.
+pub async fn set_jwt_bearer<B>(
+    Extension(auth_uri): Extension<Uri>,
+    mut request: Request<B>,
+    next: Next<B>,
+) -> Result<Response, StatusCode> {
+    let mut auth_details = None;
+
+    if let Some(bearer) = request.headers().typed_get::<Authorization<Bearer>>() {
+        auth_details = Some(make_token_request("/auth/key", bearer));
+    }
+
+    if let Some(cookie) = request.headers().typed_get::<Cookie>() {
+        auth_details = Some(make_token_request("/auth/session", cookie));
+    }
+
+    if let Some(token_request) = auth_details {
+        let response = PROXY_CLIENT
+            .call(
+                Ipv4Addr::LOCALHOST.into(),
+                &auth_uri.to_string(),
+                token_request,
+            )
+            .await
+            .expect("failed to proxy request to auth service");
+
+        let body = hyper::body::to_bytes(response.into_body()).await.unwrap();
+        let convert: Value = serde_json::from_slice(&body)
+            .expect("failed to deserialize body as JSON, did you login?");
+
+        let token = convert["token"]
+            .as_str()
+            .expect("response body should have a token");
+
+        request
+            .headers_mut()
+            .typed_insert(Authorization::bearer(token).expect("to set JWT token"));
+
+        let response = next.run(request).await;
+
+        Ok(response)
+    } else {
+        error!("No api-key bearer token or cookie found, make sure you are logged in.");
+        Err(StatusCode::UNAUTHORIZED)
+    }
+}
+
+fn make_token_request(uri: &str, header: impl Header) -> Request<Body> {
+    let mut token_request = Request::builder().uri(uri);
+    token_request
+        .headers_mut()
+        .expect("manual request to be valid")
+        .typed_insert(header);
+
+    token_request
+        .body(Body::empty())
+        .expect("manual request to be valid")
+}

deployer/src/handlers/mod.rs

@@ -4,7 +4,7 @@ use axum::extract::ws::{self, WebSocket};
 use axum::extract::{Extension, Path, Query};
 use axum::handler::Handler;
 use axum::headers::HeaderMapExt;
-use axum::middleware::from_extractor;
+use axum::middleware::{self, from_extractor};
 use axum::routing::{get, post, Router};
 use axum::{extract::BodyStream, Json};
 use bytes::BufMut;
@@ -23,7 +23,7 @@ use shuttle_common::project::ProjectName;
 use shuttle_common::storage_manager::StorageManager;
 use shuttle_common::{request_span, LogItem};
 use shuttle_service::builder::clean_crate;
-use tracing::{debug, error, field, instrument, trace};
+use tracing::{debug, error, field, instrument, trace, warn};
 use utoipa::OpenApi;
 
 use utoipa_swagger_ui::SwaggerUi;
@@ -34,8 +34,9 @@ use crate::persistence::{Deployment, Log, Persistence, ResourceManager, SecretGe
 
 use std::collections::HashMap;
 
-pub use {self::error::Error, self::error::Result};
+pub use {self::error::Error, self::error::Result, self::local::set_jwt_bearer};
 
+mod local;
 mod project;
 
 #[derive(OpenApi)]
@@ -72,88 +73,123 @@ mod project;
 )]
 pub struct ApiDoc;
 
-pub async fn make_router(
-    persistence: Persistence,
-    deployment_manager: DeploymentManager,
-    proxy_fqdn: FQDN,
-    admin_secret: String,
-    auth_uri: Uri,
+#[derive(Clone)]
+pub struct RouterBuilder {
+    router: Router,
     project_name: ProjectName,
-) -> Router {
-    Router::new()
-        // TODO: The `/swagger-ui` responds with a 303 See Other response which is followed in
-        // browsers but leads to 404 Not Found. This must be investigated.
-        .merge(SwaggerUi::new("/projects/:project_name/swagger-ui").url(
-            "/projects/:project_name/api-docs/openapi.json",
-            ApiDoc::openapi(),
-        ))
-        .route(
-            "/projects/:project_name/services",
-            get(get_services.layer(ScopedLayer::new(vec![Scope::Service]))),
-        )
-        .route(
-            "/projects/:project_name/services/:service_name",
-            get(get_service.layer(ScopedLayer::new(vec![Scope::Service])))
-                .post(create_service.layer(ScopedLayer::new(vec![Scope::ServiceCreate])))
-                .delete(stop_service.layer(ScopedLayer::new(vec![Scope::ServiceCreate]))),
-        )
-        .route(
-            "/projects/:project_name/services/:service_name/resources",
-            get(get_service_resources).layer(ScopedLayer::new(vec![Scope::Resources])),
-        )
-        .route(
-            "/projects/:project_name/deployments",
-            get(get_deployments).layer(ScopedLayer::new(vec![Scope::Service])),
-        )
-        .route(
-            "/projects/:project_name/deployments/:deployment_id",
-            get(get_deployment.layer(ScopedLayer::new(vec![Scope::Deployment])))
-                .delete(delete_deployment.layer(ScopedLayer::new(vec![Scope::DeploymentPush]))),
-        )
-        .route(
-            "/projects/:project_name/ws/deployments/:deployment_id/logs",
-            get(get_logs_subscribe.layer(ScopedLayer::new(vec![Scope::Logs]))),
-        )
-        .route(
-            "/projects/:project_name/deployments/:deployment_id/logs",
-            get(get_logs.layer(ScopedLayer::new(vec![Scope::Logs]))),
-        )
-        .route(
-            "/projects/:project_name/secrets/:service_name",
-            get(get_secrets.layer(ScopedLayer::new(vec![Scope::Secret]))),
-        )
-        .route(
-            "/projects/:project_name/clean",
-            post(clean_project.layer(ScopedLayer::new(vec![Scope::DeploymentPush]))),
-        )
-        .layer(Extension(persistence))
-        .layer(Extension(deployment_manager))
-        .layer(Extension(proxy_fqdn))
-        .layer(JwtAuthenticationLayer::new(AuthPublicKey::new(auth_uri)))
-        .layer(AdminSecretLayer::new(admin_secret))
-        // This route should be below the auth bearer since it does not need authentication
-        .route("/projects/:project_name/status", get(get_status))
-        .route_layer(from_extractor::<Metrics>())
-        .layer(
-            TraceLayer::new(|request| {
-                let account_name = request
-                    .headers()
-                    .typed_get::<XShuttleAccountName>()
-                    .unwrap_or_default();
-
-                request_span!(
-                    request,
-                    account.name = account_name.0,
-                    request.params.project_name = field::Empty,
-                    request.params.service_name = field::Empty,
-                    request.params.deployment_id = field::Empty,
-                )
-            })
-            .with_propagation()
-            .build(),
-        )
-        .route_layer(from_extractor::<project::ProjectNameGuard>())
-        .layer(Extension(project_name))
+    auth_uri: Uri,
+}
+
+impl RouterBuilder {
+    pub fn new(
+        persistence: Persistence,
+        deployment_manager: DeploymentManager,
+        proxy_fqdn: FQDN,
+        project_name: ProjectName,
+        auth_uri: Uri,
+    ) -> Self {
+        let router = Router::new()
+            // TODO: The `/swagger-ui` responds with a 303 See Other response which is followed in
+            // browsers but leads to 404 Not Found. This must be investigated.
+            .merge(SwaggerUi::new("/projects/:project_name/swagger-ui").url(
+                "/projects/:project_name/api-docs/openapi.json",
+                ApiDoc::openapi(),
+            ))
+            .route(
+                "/projects/:project_name/services",
+                get(get_services.layer(ScopedLayer::new(vec![Scope::Service]))),
+            )
+            .route(
+                "/projects/:project_name/services/:service_name",
+                get(get_service.layer(ScopedLayer::new(vec![Scope::Service])))
+                    .post(create_service.layer(ScopedLayer::new(vec![Scope::ServiceCreate])))
+                    .delete(stop_service.layer(ScopedLayer::new(vec![Scope::ServiceCreate]))),
+            )
+            .route(
+                "/projects/:project_name/services/:service_name/resources",
+                get(get_service_resources).layer(ScopedLayer::new(vec![Scope::Resources])),
+            )
+            .route(
+                "/projects/:project_name/deployments",
+                get(get_deployments).layer(ScopedLayer::new(vec![Scope::Service])),
+            )
+            .route(
+                "/projects/:project_name/deployments/:deployment_id",
+                get(get_deployment.layer(ScopedLayer::new(vec![Scope::Deployment])))
+                    .delete(delete_deployment.layer(ScopedLayer::new(vec![Scope::DeploymentPush]))),
+            )
+            .route(
+                "/projects/:project_name/ws/deployments/:deployment_id/logs",
+                get(get_logs_subscribe.layer(ScopedLayer::new(vec![Scope::Logs]))),
+            )
+            .route(
+                "/projects/:project_name/deployments/:deployment_id/logs",
+                get(get_logs.layer(ScopedLayer::new(vec![Scope::Logs]))),
+            )
+            .route(
+                "/projects/:project_name/secrets/:service_name",
+                get(get_secrets.layer(ScopedLayer::new(vec![Scope::Secret]))),
+            )
+            .route(
+                "/projects/:project_name/clean",
+                post(clean_project.layer(ScopedLayer::new(vec![Scope::DeploymentPush]))),
+            )
+            .layer(Extension(persistence))
+            .layer(Extension(deployment_manager))
+            .layer(Extension(proxy_fqdn))
+            .layer(JwtAuthenticationLayer::new(AuthPublicKey::new(
+                auth_uri.clone(),
+            )));
+
+        Self {
+            router,
+            project_name,
+            auth_uri,
+        }
+    }
+
+    pub fn with_admin_secret_layer(mut self, admin_secret: String) -> Self {
+        self.router = self.router.layer(AdminSecretLayer::new(admin_secret));
+
+        self
+    }
+
+    /// Sets an admin JWT bearer token on every request for use when running deployer locally.
+    pub fn with_local_admin_layer(mut self) -> Self {
+        warn!("Building deployer router with auth bypassed, this should only be used for local development.");
+        self.router = self
+            .router
+            .layer(middleware::from_fn(set_jwt_bearer))
+            .layer(Extension(self.auth_uri.clone()));
+
+        self
+    }
+
+    pub fn into_router(self) -> Router {
+        self.router
+            .route("/projects/:project_name/status", get(get_status))
+            .route_layer(from_extractor::<Metrics>())
+            .layer(
+                TraceLayer::new(|request| {
+                    let account_name = request
+                        .headers()
+                        .typed_get::<XShuttleAccountName>()
+                        .unwrap_or_default();
+
+                    request_span!(
+                        request,
+                        account.name = account_name.0,
+                        request.params.project_name = field::Empty,
+                        request.params.service_name = field::Empty,
+                        request.params.deployment_id = field::Empty,
+                    )
+                })
+                .with_propagation()
+                .build(),
+            )
+            .route_layer(from_extractor::<project::ProjectNameGuard>())
+            .layer(Extension(self.project_name))
+    }
 }
 
 #[instrument(skip_all)]

deployer/src/lib.rs

@@ -57,15 +57,22 @@ pub async fn start(
         deployment_manager.run_push(built).await;
     }
 
-    let router = handlers::make_router(
+    let mut builder = handlers::RouterBuilder::new(
         persistence,
         deployment_manager,
         args.proxy_fqdn,
-        args.admin_secret,
-        args.auth_uri,
         args.project,
-    )
-    .await;
+        args.auth_uri,
+    );
+
+    if args.local {
+        // If the --local flag is passed, setup an auth layer in deployer
+        builder = builder.with_local_admin_layer()
+    } else {
+        builder = builder.with_admin_secret_layer(args.admin_secret)
+    };
+
+    let router = builder.into_router();
 
     info!(address=%args.api_address, "Binding to and listening at address");