refactor: more metrics (#475)

* feat: add more metric dimensions to deployer

* feat: add more metric dimensions to gateway

* refactor: common metrics code

* refactor: forward account name

* refactor: add backend feature to deployer

* refactor: standardize naming

* refactor: cargo sort

common/Cargo.toml

@@ -9,6 +9,7 @@ description = "Common library for the shuttle platform (https://www.shuttle.rs/)
 [dependencies]
 anyhow = { version = "1.0.62", optional = true }
 async-trait = { version = "0.1.52", optional = true }
+axum = { version = "0.5.8", optional = true }
 chrono = { version = "0.4.22", features = ["serde"] }
 comfy-table = { version = "6.1.0", optional = true }
 crossterm =  { version = "0.25.0", optional = true }
@@ -25,5 +26,6 @@ uuid = { version = "1.1.1", features = ["v4", "serde"] }
 [features]
 default = ["models"]
 
+backend = ["async-trait", "axum"]
 display = ["comfy-table", "crossterm"]
 models = ["anyhow", "async-trait", "display", "http", "reqwest", "serde_json"]

common/src/backends/metrics.rs

@@ -0,0 +1,33 @@
+use std::{collections::HashMap, convert::Infallible};
+
+use async_trait::async_trait;
+use axum::extract::{FromRequest, Path, RequestParts};
+use tracing::Span;
+
+/// Used to record a bunch of metrics info
+/// The tracing layer on the server should record a `request.params.<param>` field for each parameter
+/// that should be recorded
+pub struct Metrics;
+
+#[async_trait]
+impl<B> FromRequest<B> for Metrics
+where
+    B: Send,
+{
+    type Rejection = Infallible;
+
+    async fn from_request(req: &mut RequestParts<B>) -> Result<Self, Self::Rejection> {
+        // Get path parameters if they exist
+        let Path(path): Path<HashMap<String, String>> = match req.extract().await {
+            Ok(path) => path,
+            Err(_) => return Ok(Metrics),
+        };
+
+        let span = Span::current();
+
+        for (param, value) in path {
+            span.record(format!("request.params.{param}").as_str(), value);
+        }
+        Ok(Metrics)
+    }
+}

common/src/backends/mod.rs

@@ -0,0 +1 @@
+pub mod metrics;

common/src/lib.rs

@@ -1,3 +1,5 @@
+#[cfg(feature = "backend")]
+pub mod backends;
 pub mod database;
 pub mod deployment;
 pub mod log;

deployer/Cargo.toml

@@ -45,6 +45,7 @@ uuid = { version = "1.1.2", features = ["v4"] }
 [dependencies.shuttle-common]
 version = "0.7.2"
 path = "../common"
+features = ["backend"]
 
 [dependencies.shuttle-proto]
 version = "0.7.2"

deployer/src/handlers/mod.rs

@@ -2,7 +2,7 @@ mod error;
 
 use axum::body::{Body, BoxBody};
 use axum::extract::ws::{self, WebSocket};
-use axum::extract::{Extension, Path, Query};
+use axum::extract::{Extension, MatchedPath, Path, Query};
 use axum::http::{Request, Response};
 use axum::middleware::from_extractor;
 use axum::routing::{get, Router};
@@ -13,6 +13,7 @@ use fqdn::FQDN;
 use futures::StreamExt;
 use opentelemetry::global;
 use opentelemetry_http::HeaderExtractor;
+use shuttle_common::backends::metrics::Metrics;
 use shuttle_common::models::secret;
 use shuttle_common::project::ProjectName;
 use shuttle_common::LogItem;
@@ -45,7 +46,10 @@ pub fn make_router(
             "/projects/:project_name/services/:service_name",
             get(get_service).post(post_service).delete(delete_service),
         )
-        .route("/projects/:project_name/services/:service_name/summary", get(get_service_summary))
+        .route(
+            "/projects/:project_name/services/:service_name/summary",
+            get(get_service_summary),
+        )
         .route(
             "/projects/:project_name/deployments/:deployment_id",
             get(get_deployment).delete(delete_deployment),
@@ -54,7 +58,10 @@ pub fn make_router(
             "/projects/:project_name/ws/deployments/:deployment_id/logs",
             get(get_logs_subscribe),
         )
-        .route("/projects/:project_name/deployments/:deployment_id/logs", get(get_logs))
+        .route(
+            "/projects/:project_name/deployments/:deployment_id/logs",
+            get(get_logs),
+        )
         .route(
             "/projects/:project_name/secrets/:service_name",
             get(get_secrets),
@@ -65,10 +72,34 @@ pub fn make_router(
         .layer(RequireAuthorizationLayer::bearer(&admin_secret))
         // This route should be below the auth bearer since it does not need authentication
         .route("/projects/:project_name/status", get(get_status))
+        .route_layer(from_extractor::<Metrics>())
         .layer(
             TraceLayer::new_for_http()
                 .make_span_with(|request: &Request<Body>| {
-                    let span = debug_span!("request", http.uri = %request.uri(), http.method = %request.method(), http.status_code = field::Empty);
+                    let path = if let Some(path) = request.extensions().get::<MatchedPath>() {
+                        path.as_str()
+                    } else {
+                        ""
+                    };
+
+                    let account_name = request
+                        .headers()
+                        .get("X-Shuttle-Account-Name")
+                        .map(|value| value.to_str().unwrap_or_default());
+
+                    let span = debug_span!(
+                        "request",
+                        http.uri = %request.uri(),
+                        http.method = %request.method(),
+                        http.status_code = field::Empty,
+                        account.name = account_name,
+                        // A bunch of extra things for metrics
+                        // Should be able to make this clearer once `Valuable` support lands in tracing
+                        request.path = path,
+                        request.params.project_name = field::Empty,
+                        request.params.service_name = field::Empty,
+                        request.params.deployment_id = field::Empty,
+                    );
                     let parent_context = global::get_text_map_propagator(|propagator| {
                         propagator.extract(&HeaderExtractor(request.headers()))
                     });
@@ -79,7 +110,10 @@ pub fn make_router(
                 .on_response(
                     |response: &Response<BoxBody>, latency: Duration, span: &Span| {
                         span.record("http.status_code", &response.status().as_u16());
-                        debug!(latency = format_args!("{} ns", latency.as_nanos()), "finished processing request");
+                        debug!(
+                            latency = format_args!("{} ns", latency.as_nanos()),
+                            "finished processing request"
+                        );
                     },
                 ),
         )

gateway/Cargo.toml

@@ -7,24 +7,19 @@ publish = false
 [dependencies]
 acme2 = "0.5.1"
 async-trait = "0.1.52"
-
 axum = { version = "0.5.8", features = [ "headers" ] }
 axum-server = { version = "0.4.4", features = [ "tls-rustls" ] }
-
 base64 = "0.13"
 bollard = "0.13"
 chrono = "0.4"
 clap = { version = "4.0.0", features = [ "derive" ] }
-
 fqdn = "0.2.2"
-
 futures = "0.3.21"
 http = "0.2.8"
 hyper = { version = "0.14.19", features = [ "stream" ] }
 # not great, but waiting for WebSocket changes to be merged
 hyper-reverse-proxy = { git = "https://github.com/chesedo/hyper-reverse-proxy", branch = "bug/host_header" }
 instant-acme = "0.1.0"
-
 lazy_static = "1.4.0"
 once_cell = "1.14.0"
 opentelemetry = { version = "0.18.0", features = ["rt-tokio"] }
@@ -50,6 +45,7 @@ uuid = { version = "1.2.1", features = [ "v4" ] }
 [dependencies.shuttle-common]
 version = "0.7.2"
 path = "../common"
+features = ["backend"]
 
 [dev-dependencies]
 anyhow = "1"

gateway/src/api/latest.rs

@@ -4,8 +4,9 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use axum::body::{Body, BoxBody};
-use axum::extract::{Extension, Path};
+use axum::extract::{Extension, MatchedPath, Path};
 use axum::http::Request;
+use axum::middleware::from_extractor;
 use axum::response::Response;
 use axum::routing::{any, get, post};
 use axum::{Json as AxumJson, Router};
@@ -14,6 +15,7 @@ use futures::Future;
 use http::StatusCode;
 use instant_acme::{AccountCredentials, ChallengeType};
 use serde::{Deserialize, Serialize};
+use shuttle_common::backends::metrics::Metrics;
 use shuttle_common::models::error::ErrorKind;
 use shuttle_common::models::{project, user};
 use tokio::sync::mpsc::Sender;
@@ -151,10 +153,10 @@ async fn delete_project(
 
 async fn route_project(
     Extension(service): Extension<Arc<GatewayService>>,
-    ScopedUser { scope, .. }: ScopedUser,
+    scoped_user: ScopedUser,
     req: Request<Body>,
 ) -> Result<Response<Body>, Error> {
-    service.route(&scope, req).await
+    service.route(&scoped_user, req).await
 }
 
 async fn get_status(Extension(sender): Extension<Sender<BoxedTask>>) -> Response<Body> {
@@ -310,15 +312,35 @@ impl ApiBuilder {
     }
 
     pub fn with_default_traces(mut self) -> Self {
-        self.router = self.router.layer(
+        self.router = self.router.route_layer(from_extractor::<Metrics>()).layer(
             TraceLayer::new_for_http()
                 .make_span_with(|request: &Request<Body>| {
-                    debug_span!("request", http.uri = %request.uri(), http.method = %request.method(), http.status_code = field::Empty, account.name = field::Empty, account.project = field::Empty)
+                    let path = if let Some(path) = request.extensions().get::<MatchedPath>() {
+                        path.as_str()
+                    } else {
+                        ""
+                    };
+
+                    debug_span!(
+                        "request",
+                        http.uri = %request.uri(),
+                        http.method = %request.method(),
+                        http.status_code = field::Empty,
+                        account.name = field::Empty,
+                        // A bunch of extra things for metrics
+                        // Should be able to make this clearer once `Valuable` support lands in tracing
+                        request.path = path,
+                        request.params.project_name = field::Empty,
+                        request.params.account_name = field::Empty,
+                    )
                 })
                 .on_response(
                     |response: &Response<BoxBody>, latency: Duration, span: &Span| {
                         span.record("http.status_code", response.status().as_u16());
-                        debug!(latency = format_args!("{} ns", latency.as_nanos()), "finished processing request");
+                        debug!(
+                            latency = format_args!("{} ns", latency.as_nanos()),
+                            "finished processing request"
+                        );
                     },
                 ),
         );
@@ -330,11 +352,11 @@ impl ApiBuilder {
             .router
             .route("/", get(get_status))
             .route(
-                "/projects/:project",
+                "/projects/:project_name",
                 get(get_project).delete(delete_project).post(post_project),
             )
             .route("/users/:account_name", get(get_user).post(post_user))
-            .route("/projects/:project/*any", any(route_project))
+            .route("/projects/:project_name/*any", any(route_project))
             .route("/admin/revive", post(revive_projects));
         self
     }

gateway/src/auth.rs

@@ -247,8 +247,6 @@ where
                 .unwrap(),
         };
 
-        // Record current project for tracing purposes
-        Span::current().record("account.project", &scope.to_string());
         if user.is_super_user() || user.projects.contains(&scope) {
             Ok(Self { user, scope })
         } else {

gateway/src/service.rs

@@ -8,6 +8,7 @@ use axum::response::Response;
 use bollard::network::ListNetworksOptions;
 use bollard::{Docker, API_DEFAULT_VERSION};
 use fqdn::Fqdn;
+use http::HeaderValue;
 use hyper::client::connect::dns::GaiResolver;
 use hyper::client::HttpConnector;
 use hyper::Client;
@@ -25,7 +26,7 @@ use tracing_opentelemetry::OpenTelemetrySpanExt;
 
 use crate::acme::CustomDomain;
 use crate::args::ContextArgs;
-use crate::auth::{Key, Permissions, User};
+use crate::auth::{Key, Permissions, ScopedUser, User};
 use crate::project::Project;
 use crate::task::TaskBuilder;
 use crate::{AccountName, DockerContext, Error, ErrorKind, ProjectName};
@@ -205,9 +206,10 @@ impl GatewayService {
 
     pub async fn route(
         &self,
-        project_name: &ProjectName,
+        scoped_user: &ScopedUser,
         mut req: Request<Body>,
     ) -> Result<Response<Body>, Error> {
+        let project_name = &scoped_user.scope;
         let target_ip = self
             .find_project(project_name)
             .await?
@@ -223,9 +225,15 @@ impl GatewayService {
 
         debug!(target_url, "routing control");
 
+        let headers = req.headers_mut();
+        headers.append(
+            "X-Shuttle-Account-Name",
+            HeaderValue::from_str(&scoped_user.user.name.to_string()).unwrap(),
+        );
+
         let cx = Span::current().context();
         global::get_text_map_propagator(|propagator| {
-            propagator.inject_context(&cx, &mut HeaderInjector(req.headers_mut()))
+            propagator.inject_context(&cx, &mut HeaderInjector(headers))
         });
 
         let resp = PROXY_CLIENT