From 8e7a51b6d8ea8bdf8fd3947b7a5109bb25f6a858 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 14 Mar 2025 15:23:15 -0700 Subject: [PATCH 01/39] wip --- Cargo.toml | 2 +- src/health.rs | 63 ++++++++++++++ src/main.rs | 19 +++-- src/metrics.rs | 12 +-- src/proxy.rs | 223 ++++++++++++++++++++++++++++++++----------------- 5 files changed, 230 insertions(+), 89 deletions(-) create mode 100644 src/health.rs diff --git a/Cargo.toml b/Cargo.toml index 85c66977..efc3a93b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ moka = { version = "0.12.10", features = ["sync"] } reqwest = "0.12.5" http = "1.1.0" dotenv = "0.15.0" -tower = "0.4.13" +tower = { version = "0.4.13", features = ["filter"] } http-body = "1.0.1" http-body-util = "0.1.2" hyper = { version = "1.4.1", features = ["full"] } diff --git a/src/health.rs b/src/health.rs new file mode 100644 index 00000000..82c94a84 --- /dev/null +++ b/src/health.rs @@ -0,0 +1,63 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use futures::FutureExt as _; +use jsonrpsee::{ + core::BoxError, + http_client::{HttpBody, HttpRequest, HttpResponse}, +}; +use tower::{Layer, Service, util::Either}; + +pub(crate) struct HealthLayer; + +impl Layer for HealthLayer { + type Service = HealthService; + + fn layer(&self, service: S) -> Self::Service { + HealthService(service) + } +} + +#[derive(Clone)] +pub struct HealthService(S); + +impl Service> for HealthService +where + S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, + S::Response: 'static, + S::Error: Into + 'static, + S::Future: Send + 'static, +{ + type Response = HttpResponse; + type Error = BoxError; + type Future = Either< + Pin> + Send + 'static>>, + S::Future, + >; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.0.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, request: HttpRequest) -> Self::Future { + if request.uri().path() == "/healthz" { + Either::A(Self::healthz().boxed()) + } else { + Either::B(self.0.call(request)) + } + } +} + +impl HealthService +where + S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, + S::Response: 'static, + S::Error: Into + 'static, + S::Future: Send + 'static, +{ + async fn healthz() -> Result { + Ok(HttpResponse::new(HttpBody::from("OK"))) + } +} diff --git a/src/main.rs b/src/main.rs index 95bf4579..6812e182 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use ::tracing::{Level, error, info}; use clap::{Parser, Subcommand, arg}; use client::{BuilderArgs, ExecutionClient, L2ClientArgs}; use debug_api::DebugClient; +use health::HealthLayer; use metrics::init_metrics; use std::net::SocketAddr; use tracing::init_tracing; @@ -27,6 +28,7 @@ use tokio::signal::unix::{SignalKind, signal as unix_signal}; mod auth_layer; mod client; mod debug_api; +mod health; #[cfg(all(feature = "integration", test))] mod integration; mod metrics; @@ -225,13 +227,16 @@ async fn main() -> eyre::Result<()> { // Build and start the server info!("Starting server on :{}", args.rpc_port); - let service_builder = tower::ServiceBuilder::new().layer(ProxyLayer::new( - l2_client_args.l2_url, - l2_auth_jwt, - builder_args.builder_url, - builder_auth_jwt, - metrics, - )); + let service_builder = tower::ServiceBuilder::new() + .layer(HealthLayer) + .layer(ProxyLayer::new( + l2_client_args.l2_url, + l2_auth_jwt, + builder_args.builder_url, + builder_auth_jwt, + metrics, + )) + .layer(HealthLayer); let server = Server::builder() .set_http_middleware(service_builder) diff --git a/src/metrics.rs b/src/metrics.rs index be5b0579..383c2ba9 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -59,12 +59,12 @@ impl ServerMetrics { rpc_status_code: Option, method: String, ) { - counter!("rpc.l2_response_count", - "http_status_code" => http_status_code, - "rpc_status_code" => rpc_status_code.unwrap_or("".to_string()), - "method" => method, - ) - .increment(1); + let mut labels = vec![("http_status_code", http_status_code), ("method", method)]; + if let Some(rpc_status_code) = rpc_status_code { + labels.push(("rpc_status_code", rpc_status_code)); + } + + counter!("rpc.l2_response_count", &labels).increment(1); } } diff --git a/src/proxy.rs b/src/proxy.rs index 44fba2d6..36b1a1c1 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -3,6 +3,7 @@ use crate::metrics::ServerMetrics; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; use flate2::read::GzDecoder; +use futures_util::future::FutureExt; use http::header::AUTHORIZATION; use http::response::Parts; use http::{StatusCode, Uri}; @@ -19,7 +20,7 @@ use std::task::{Context, Poll}; use std::time::{Duration, Instant}; use std::{future::Future, pin::Pin}; use tower::{Layer, Service}; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, info, instrument, warn}; const MULTIPLEX_METHODS: [&str; 4] = [ "engine_", @@ -98,108 +99,180 @@ pub struct ProxyService { metrics: Option>, } -impl Service> for ProxyService +impl ProxyService where - S: Service, Response = HttpResponse> + Send + Clone + 'static, + S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, S::Response: 'static, S::Error: Into + 'static, S::Future: Send + 'static, { - type Response = S::Response; - type Error = BoxError; - type Future = - Pin> + Send + 'static>>; + #[instrument(skip(self), err)] + async fn call(&mut self, req: HttpRequest) -> Result { + #[derive(serde::Deserialize, Debug)] + struct RpcRequest<'a> { + #[serde(borrow)] + method: &'a str, + } - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.poll_ready(cx).map_err(Into::into) - } + let (parts, body) = req.into_parts(); + let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; + + // Deserialize the bytes to find the method + let method = serde_json::from_slice::(&body_bytes)? + .method + .to_string(); + + if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { + if FORWARD_REQUESTS.contains(&method.as_str()) { + let builder_client = self.client.clone(); + let builder_req = + HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); + let builder_method = method.clone(); + let builder_metrics = self.metrics.clone(); + let builder_auth_uri = self.builder_auth_uri.clone(); + let builder_auth_secret = self.builder_auth_secret.clone(); + tokio::spawn(async move { + let _ = forward_request( + &builder_client, + builder_req, + builder_method, + builder_auth_uri, + builder_auth_secret, + builder_metrics, + PayloadSource::Builder, + ) + .await; + }); - fn call(&mut self, req: HttpRequest) -> Self::Future { - if req.uri().path() == "/healthz" { - return Box::pin(async { Ok(Self::Response::new(HttpBody::from("OK"))) }); + let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); + forward_request( + &self.client, + l2_req, + method, + self.l2_auth_uri.clone(), + self.l2_auth_secret, + self.metrics.clone(), + PayloadSource::L2, + ) + .await + } else { + let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); + } + } else { + let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + forward_request( + &self.client, + req, + method, + self.l2_auth_uri.clone(), + self.l2_auth_secret.clone(), + self.metrics.clone(), + PayloadSource::L2, + ) + .await } - let client = self.client.clone(); - let mut inner = self.inner.clone(); - let builder_uri = self.builder_auth_uri.clone(); - let builder_secret = self.builder_auth_secret; - let l2_uri = self.l2_auth_uri.clone(); - let l2_secret = self.l2_auth_secret; - let metrics = self.metrics.clone(); + self.inner.call(req).await.map_err(|e| e.into()) + } + #[instrument(skip(self), err)] + async fn proxy(&mut self, req: HttpRequest) -> Result { #[derive(serde::Deserialize, Debug)] struct RpcRequest<'a> { #[serde(borrow)] method: &'a str, } - let fut = async move { - let (parts, body) = req.into_parts(); - let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; - - // Deserialize the bytes to find the method - let method = serde_json::from_slice::(&body_bytes)? - .method - .to_string(); - - if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { - if FORWARD_REQUESTS.contains(&method.as_str()) { - let builder_client = client.clone(); - let builder_req = - HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); - let builder_method = method.clone(); - let builder_metrics = metrics.clone(); - tokio::spawn(async move { - let _ = forward_request( - builder_client, - builder_req, - builder_method, - builder_uri, - builder_secret, - builder_metrics, - PayloadSource::Builder, - ) - .await; - }); - - let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - forward_request( - client, - l2_req, - method, - l2_uri, - l2_secret, - metrics, - PayloadSource::L2, + let (parts, body) = req.into_parts(); + let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; + + // Deserialize the bytes to find the method + let method = serde_json::from_slice::(&body_bytes)? + .method + .to_string(); + + if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { + if FORWARD_REQUESTS.contains(&method.as_str()) { + let builder_client = self.client.clone(); + let builder_req = + HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); + let builder_method = method.clone(); + let builder_metrics = self.metrics.clone(); + let builder_auth_uri = self.builder_auth_uri.clone(); + let builder_auth_secret = self.builder_auth_secret.clone(); + tokio::spawn(async move { + let _ = forward_request( + &builder_client, + builder_req, + builder_method, + builder_auth_uri, + builder_auth_secret, + builder_metrics, + PayloadSource::Builder, ) - .await - } else { - let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - inner.call(req).await.map_err(|e| e.into()) - } - } else { - let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + .await; + }); + + let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); forward_request( - client, - req, + &self.client, + l2_req, method, - l2_uri, - l2_secret, - metrics, + self.l2_auth_uri.clone(), + self.l2_auth_secret, + self.metrics.clone(), PayloadSource::L2, ) .await + } else { + let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); } - }; - Box::pin(fut) + } else { + let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + forward_request( + &self.client, + req, + method, + self.l2_auth_uri.clone(), + self.l2_auth_secret.clone(), + self.metrics.clone(), + PayloadSource::L2, + ) + .await + } + + self.inner.call(req).await.map_err(|e| e.into()) + } +} + +impl Service> for ProxyService +where + S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, + S::Response: 'static, + S::Error: Into + 'static, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = BoxError; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, req: HttpRequest) -> Self::Future { + self.call(req).boxed() } } /// Forwards an HTTP request to the `authrpc``, attaching the provided JWT authorization. async fn forward_request( - client: Client, HttpBody>, + client: &Client, HttpBody>, mut req: http::Request, method: String, uri: Uri, From 39ef56e8285974d7a7b33431f5b908b2758a325d Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 14 Mar 2025 16:15:38 -0700 Subject: [PATCH 02/39] wip --- src/health.rs | 1 + src/main.rs | 24 ++-- src/proxy.rs | 325 +++++++++++++++++++++++++++----------------------- 3 files changed, 187 insertions(+), 163 deletions(-) diff --git a/src/health.rs b/src/health.rs index 82c94a84..9592ed6f 100644 --- a/src/health.rs +++ b/src/health.rs @@ -10,6 +10,7 @@ use jsonrpsee::{ }; use tower::{Layer, Service, util::Either}; +/// A [`Layer`] that filters out /healthz requests and responds with a 200 OK. pub(crate) struct HealthLayer; impl Layer for HealthLayer { diff --git a/src/main.rs b/src/main.rs index 6812e182..e5a2ac79 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,7 @@ use hyper::{Request, Response, server::conn::http1}; use hyper_util::rt::TokioIo; use jsonrpsee::RpcModule; use jsonrpsee::http_client::HttpBody; -use jsonrpsee::server::Server; +use jsonrpsee::server::{RpcServiceBuilder, Server}; use metrics_exporter_prometheus::PrometheusHandle; use proxy::ProxyLayer; use server::{ExecutionMode, PayloadSource, RollupBoostServer}; @@ -227,19 +227,19 @@ async fn main() -> eyre::Result<()> { // Build and start the server info!("Starting server on :{}", args.rpc_port); - let service_builder = tower::ServiceBuilder::new() - .layer(HealthLayer) - .layer(ProxyLayer::new( - l2_client_args.l2_url, - l2_auth_jwt, - builder_args.builder_url, - builder_auth_jwt, - metrics, - )) - .layer(HealthLayer); + let http_middleware = tower::ServiceBuilder::new().layer(HealthLayer); + + let rpc_middleware = RpcServiceBuilder::new().layer(ProxyLayer::new( + l2_client_args.l2_url, + l2_auth_jwt, + builder_args.builder_url, + builder_auth_jwt, + metrics, + )); let server = Server::builder() - .set_http_middleware(service_builder) + .set_http_middleware(http_middleware) + .set_rpc_middleware(rpc_middleware) .build(format!("{}:{}", args.rpc_host, args.rpc_port).parse::()?) .await?; let handle = server.start(module); diff --git a/src/proxy.rs b/src/proxy.rs index 36b1a1c1..22c3961a 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -12,8 +12,12 @@ use hyper_rustls::HttpsConnector; use hyper_util::client::legacy::Client; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::rt::TokioExecutor; +use jsonrpsee::MethodResponse; use jsonrpsee::core::{BoxError, http_helpers}; use jsonrpsee::http_client::{HttpBody, HttpRequest, HttpResponse}; +use jsonrpsee::server::middleware::rpc::RpcServiceT; +use jsonrpsee::types::Request; +use opentelemetry::trace::SpanKind; use std::io::Read; use std::sync::Arc; use std::task::{Context, Poll}; @@ -99,198 +103,217 @@ pub struct ProxyService { metrics: Option>, } -impl ProxyService +// impl ProxyService +// where +// S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, +// S::Response: 'static, +// S::Error: Into + 'static, +// S::Future: Send + 'static, +// { +// #[instrument(skip(self), err)] +// async fn proxy(self, req: HttpRequest) -> Result { +// } +// } + +impl Service> for ProxyService where S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, S::Response: 'static, S::Error: Into + 'static, S::Future: Send + 'static, { - #[instrument(skip(self), err)] - async fn call(&mut self, req: HttpRequest) -> Result { - #[derive(serde::Deserialize, Debug)] - struct RpcRequest<'a> { - #[serde(borrow)] - method: &'a str, - } + type Response = S::Response; + type Error = BoxError; + type Future = + Pin> + Send + 'static>>; - let (parts, body) = req.into_parts(); - let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; - - // Deserialize the bytes to find the method - let method = serde_json::from_slice::(&body_bytes)? - .method - .to_string(); - - if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { - if FORWARD_REQUESTS.contains(&method.as_str()) { - let builder_client = self.client.clone(); - let builder_req = - HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); - let builder_method = method.clone(); - let builder_metrics = self.metrics.clone(); - let builder_auth_uri = self.builder_auth_uri.clone(); - let builder_auth_secret = self.builder_auth_secret.clone(); - tokio::spawn(async move { - let _ = forward_request( - &builder_client, - builder_req, - builder_method, - builder_auth_uri, - builder_auth_secret, - builder_metrics, - PayloadSource::Builder, - ) - .await; - }); + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + } - let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - forward_request( - &self.client, - l2_req, - method, - self.l2_auth_uri.clone(), - self.l2_auth_secret, - self.metrics.clone(), - PayloadSource::L2, - ) - .await - } else { - let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - } - } else { - let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - forward_request( - &self.client, - req, - method, - self.l2_auth_uri.clone(), - self.l2_auth_secret.clone(), - self.metrics.clone(), - PayloadSource::L2, - ) - .await + fn call(&mut self, req: HttpRequest) -> Self::Future { + // // self.proxy(req).boxed() + // #[derive(serde::Deserialize, Debug)] + // struct RpcRequest<'a> { + // #[serde(borrow)] + // method: &'a str, + // } + // + // let (parts, body) = req.into_parts(); + // let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; + // + // // Deserialize the bytes to find the method + // let method = serde_json::from_slice::(&body_bytes)? + // .method + // .to_string(); + // + // if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { + // if FORWARD_REQUESTS.contains(&method.as_str()) { + // let builder_client = self.client.clone(); + // let builder_req = + // HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); + // let builder_method = method.clone(); + // let builder_metrics = self.metrics.clone(); + // let builder_auth_uri = self.builder_auth_uri.clone(); + // let builder_auth_secret = self.builder_auth_secret.clone(); + // tokio::spawn(async move { + // let _ = forward_request( + // &builder_client, + // builder_req, + // builder_method, + // builder_auth_uri, + // builder_auth_secret, + // builder_metrics, + // PayloadSource::Builder, + // ) + // .await; + // }); + // + // let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + // info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); + // forward_request( + // &self.client, + // l2_req, + // method, + // self.l2_auth_uri.clone(), + // self.l2_auth_secret, + // self.metrics.clone(), + // PayloadSource::L2, + // ) + // .await + // } else { + // let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + // info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); + // } + // } else { + // let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + // forward_request( + // &self.client, + // req, + // method, + // self.l2_auth_uri.clone(), + // self.l2_auth_secret.clone(), + // self.metrics.clone(), + // PayloadSource::L2, + // ) + // .await + // } + // + // self.inner.call(req).await.map_err(|e| e.into()) + if req.uri().path() == "/healthz" { + return Box::pin(async { Ok(Self::Response::new(HttpBody::from("OK"))) }); } - self.inner.call(req).await.map_err(|e| e.into()) - } + let client = self.client.clone(); + let mut inner = self.inner.clone(); + let builder_uri = self.builder_auth_uri.clone(); + let builder_secret = self.builder_auth_secret; + let l2_uri = self.l2_auth_uri.clone(); + let l2_secret = self.l2_auth_secret; + let metrics = self.metrics.clone(); - #[instrument(skip(self), err)] - async fn proxy(&mut self, req: HttpRequest) -> Result { #[derive(serde::Deserialize, Debug)] struct RpcRequest<'a> { #[serde(borrow)] method: &'a str, } - let (parts, body) = req.into_parts(); - let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; - - // Deserialize the bytes to find the method - let method = serde_json::from_slice::(&body_bytes)? - .method - .to_string(); - - if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { - if FORWARD_REQUESTS.contains(&method.as_str()) { - let builder_client = self.client.clone(); - let builder_req = - HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); - let builder_method = method.clone(); - let builder_metrics = self.metrics.clone(); - let builder_auth_uri = self.builder_auth_uri.clone(); - let builder_auth_secret = self.builder_auth_secret.clone(); - tokio::spawn(async move { - let _ = forward_request( - &builder_client, - builder_req, - builder_method, - builder_auth_uri, - builder_auth_secret, - builder_metrics, - PayloadSource::Builder, - ) - .await; - }); + let fut = async move { + let (parts, body) = req.into_parts(); + let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; + + // Deserialize the bytes to find the method + let method = serde_json::from_slice::(&body_bytes)? + .method + .to_string(); + + if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { + if FORWARD_REQUESTS.contains(&method.as_str()) { + let builder_client = client.clone(); + let builder_req = + HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); + let builder_method = method.clone(); + let builder_metrics = metrics.clone(); + tokio::spawn(async move { + let _ = forward_request( + builder_client, + builder_req, + builder_method, + builder_uri, + builder_secret, + builder_metrics, + PayloadSource::Builder, + ) + .await; + }); - let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); + let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); + forward_request( + client, + l2_req, + method, + l2_uri, + l2_secret, + metrics, + PayloadSource::L2, + ) + .await + } else { + let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); + inner.call(req).await.map_err(|e| e.into()) + } + } else { + let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); forward_request( - &self.client, - l2_req, + client, + req, method, - self.l2_auth_uri.clone(), - self.l2_auth_secret, - self.metrics.clone(), + l2_uri, + l2_secret, + metrics, PayloadSource::L2, ) .await - } else { - let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); } - } else { - let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - forward_request( - &self.client, - req, - method, - self.l2_auth_uri.clone(), - self.l2_auth_secret.clone(), - self.metrics.clone(), - PayloadSource::L2, - ) - .await - } - - self.inner.call(req).await.map_err(|e| e.into()) + }; + Box::pin(fut) } } -impl Service> for ProxyService +impl RpcServiceT<'_> for ProxyService where - S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, - S::Response: 'static, - S::Error: Into + 'static, - S::Future: Send + 'static, + for<'a> S: RpcServiceT<'a> + Send + Sync + Clone + 'static, { - type Response = S::Response; - type Error = BoxError; - type Future = - Pin> + Send + 'static>>; + type Future = Pin + Send + 'static>>; - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.poll_ready(cx).map_err(Into::into) - } - - fn call(&mut self, req: HttpRequest) -> Self::Future { - self.call(req).boxed() + fn call(&self, request: Request<'_>) -> Self::Future { + todo!() } } /// Forwards an HTTP request to the `authrpc``, attaching the provided JWT authorization. +#[instrument( + skip(client, req, auth, metrics), + fields(otel.kind = ?SpanKind::Client), + err +)] async fn forward_request( - client: &Client, HttpBody>, + client: Client, HttpBody>, mut req: http::Request, method: String, - uri: Uri, + url: Uri, auth: JwtSecret, metrics: Option>, - source: PayloadSource, + target: PayloadSource, ) -> Result, BoxError> { let start = Instant::now(); - *req.uri_mut() = uri.clone(); + *req.uri_mut() = url.clone(); req.headers_mut() .insert(AUTHORIZATION, secret_to_bearer_header(&auth)); - debug!( - target: "proxy::forward_request", - url = ?uri, - ?method, - ?req, - ); + debug!("forwarding request to {}", url); match client.request(req).await { Ok(resp) => { @@ -318,7 +341,7 @@ async fn forward_request( metrics, method, start.elapsed(), - source, + target, ) .await; }); @@ -332,7 +355,7 @@ async fn forward_request( error!( target: "proxy::call", message = "error forwarding request", - url = ?uri, + url = ?url, method = %method, error = %e, ); @@ -342,7 +365,7 @@ async fn forward_request( None, method, start.elapsed(), - source, + target, ) .await; Err(e.into()) From e49f054f64754a3dc9b70dbaf7abb440a471273d Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 14 Mar 2025 17:59:52 -0700 Subject: [PATCH 03/39] wip --- src/main.rs | 21 +++--- src/proxy.rs | 182 ++++++++++----------------------------------------- 2 files changed, 43 insertions(+), 160 deletions(-) diff --git a/src/main.rs b/src/main.rs index e5a2ac79..775cf9ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,7 @@ use hyper::{Request, Response, server::conn::http1}; use hyper_util::rt::TokioIo; use jsonrpsee::RpcModule; use jsonrpsee::http_client::HttpBody; -use jsonrpsee::server::{RpcServiceBuilder, Server}; +use jsonrpsee::server::Server; use metrics_exporter_prometheus::PrometheusHandle; use proxy::ProxyLayer; use server::{ExecutionMode, PayloadSource, RollupBoostServer}; @@ -227,19 +227,18 @@ async fn main() -> eyre::Result<()> { // Build and start the server info!("Starting server on :{}", args.rpc_port); - let http_middleware = tower::ServiceBuilder::new().layer(HealthLayer); - - let rpc_middleware = RpcServiceBuilder::new().layer(ProxyLayer::new( - l2_client_args.l2_url, - l2_auth_jwt, - builder_args.builder_url, - builder_auth_jwt, - metrics, - )); + let http_middleware = tower::ServiceBuilder::new() + .layer(HealthLayer) + .layer(ProxyLayer::new( + l2_client_args.l2_url, + l2_auth_jwt, + builder_args.builder_url, + builder_auth_jwt, + metrics, + )); let server = Server::builder() .set_http_middleware(http_middleware) - .set_rpc_middleware(rpc_middleware) .build(format!("{}:{}", args.rpc_host, args.rpc_port).parse::()?) .await?; let handle = server.start(module); diff --git a/src/proxy.rs b/src/proxy.rs index 22c3961a..9c37f8ad 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -103,18 +103,6 @@ pub struct ProxyService { metrics: Option>, } -// impl ProxyService -// where -// S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, -// S::Response: 'static, -// S::Error: Into + 'static, -// S::Future: Send + 'static, -// { -// #[instrument(skip(self), err)] -// async fn proxy(self, req: HttpRequest) -> Result { -// } -// } - impl Service> for ProxyService where S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, @@ -132,78 +120,6 @@ where } fn call(&mut self, req: HttpRequest) -> Self::Future { - // // self.proxy(req).boxed() - // #[derive(serde::Deserialize, Debug)] - // struct RpcRequest<'a> { - // #[serde(borrow)] - // method: &'a str, - // } - // - // let (parts, body) = req.into_parts(); - // let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; - // - // // Deserialize the bytes to find the method - // let method = serde_json::from_slice::(&body_bytes)? - // .method - // .to_string(); - // - // if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { - // if FORWARD_REQUESTS.contains(&method.as_str()) { - // let builder_client = self.client.clone(); - // let builder_req = - // HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); - // let builder_method = method.clone(); - // let builder_metrics = self.metrics.clone(); - // let builder_auth_uri = self.builder_auth_uri.clone(); - // let builder_auth_secret = self.builder_auth_secret.clone(); - // tokio::spawn(async move { - // let _ = forward_request( - // &builder_client, - // builder_req, - // builder_method, - // builder_auth_uri, - // builder_auth_secret, - // builder_metrics, - // PayloadSource::Builder, - // ) - // .await; - // }); - // - // let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - // info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - // forward_request( - // &self.client, - // l2_req, - // method, - // self.l2_auth_uri.clone(), - // self.l2_auth_secret, - // self.metrics.clone(), - // PayloadSource::L2, - // ) - // .await - // } else { - // let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - // info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - // } - // } else { - // let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - // forward_request( - // &self.client, - // req, - // method, - // self.l2_auth_uri.clone(), - // self.l2_auth_secret.clone(), - // self.metrics.clone(), - // PayloadSource::L2, - // ) - // .await - // } - // - // self.inner.call(req).await.map_err(|e| e.into()) - if req.uri().path() == "/healthz" { - return Box::pin(async { Ok(Self::Response::new(HttpBody::from("OK"))) }); - } - let client = self.client.clone(); let mut inner = self.inner.clone(); let builder_uri = self.builder_auth_uri.clone(); @@ -282,17 +198,6 @@ where } } -impl RpcServiceT<'_> for ProxyService -where - for<'a> S: RpcServiceT<'a> + Send + Sync + Clone + 'static, -{ - type Future = Pin + Send + 'static>>; - - fn call(&self, request: Request<'_>) -> Self::Future { - todo!() - } -} - /// Forwards an HTTP request to the `authrpc``, attaching the provided JWT authorization. #[instrument( skip(client, req, auth, metrics), @@ -308,69 +213,48 @@ async fn forward_request( metrics: Option>, target: PayloadSource, ) -> Result, BoxError> { + debug!("forwarding {} to {}", method, target); let start = Instant::now(); + *req.uri_mut() = url.clone(); req.headers_mut() .insert(AUTHORIZATION, secret_to_bearer_header(&auth)); - debug!("forwarding request to {}", url); + let res = client.request(req).await?; - match client.request(req).await { - Ok(resp) => { - let (parts, body) = resp.into_parts(); - let body_bytes = body - .collect() - .await - .map_err(|e| { - error!( - target: "proxy::forward_request", - message = "error collecting body", - error = %e, - ); - e - })? - .to_bytes() - .to_vec(); - let parts_clone = parts.clone(); - let body_bytes_clone = body_bytes.clone(); - - tokio::spawn(async move { - let _ = process_response( - parts_clone, - body_bytes_clone, - metrics, - method, - start.elapsed(), - target, - ) - .await; - }); - - Ok(http::Response::from_parts( - parts, - HttpBody::from(body_bytes), - )) - } - Err(e) => { + let (parts, body) = res.into_parts(); + let body_bytes = body + .collect() + .await + .map_err(|e| { error!( - target: "proxy::call", - message = "error forwarding request", - url = ?url, - method = %method, + target: "proxy::forward_request", + message = "error collecting body", error = %e, ); - record_metrics( - metrics, - StatusCode::INTERNAL_SERVER_ERROR.to_string(), - None, - method, - start.elapsed(), - target, - ) - .await; - Err(e.into()) - } - } + e + })? + .to_bytes() + .to_vec(); + let parts_clone = parts.clone(); + let body_bytes_clone = body_bytes.clone(); + + tokio::spawn(async move { + let _ = process_response( + parts_clone, + body_bytes_clone, + metrics, + method, + start.elapsed(), + target, + ) + .await; + }); + + Ok(http::Response::from_parts( + parts, + HttpBody::from(body_bytes), + )) } async fn process_response( From f77ff561f4f6b877c5ef7b018b6a150b00ddf82f Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 10:54:14 -0700 Subject: [PATCH 04/39] wip --- src/client/auth.rs | 82 ++++++++++ src/client/http.rs | 142 +++++++++++++++++ src/client/mod.rs | 3 + src/{client.rs => client/rpc.rs} | 28 ++-- src/integration/mod.rs | 2 +- src/main.rs | 11 +- src/proxy.rs | 252 ++++--------------------------- src/server.rs | 15 +- 8 files changed, 284 insertions(+), 251 deletions(-) create mode 100644 src/client/auth.rs create mode 100644 src/client/http.rs create mode 100644 src/client/mod.rs rename src/{client.rs => client/rpc.rs} (92%) diff --git a/src/client/auth.rs b/src/client/auth.rs new file mode 100644 index 00000000..86723fc4 --- /dev/null +++ b/src/client/auth.rs @@ -0,0 +1,82 @@ +// From reth_rpc_layer +use alloy_rpc_types_engine::{Claims, JwtSecret}; +use http::{HeaderValue, header::AUTHORIZATION}; +use std::{ + task::{Context, Poll}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tower::{Layer, Service}; + +/// A layer that adds a new JWT token to every request using `AuthClientService`. +#[derive(Clone, Debug)] +pub struct AuthClientLayer { + secret: JwtSecret, +} + +impl AuthClientLayer { + /// Create a new `AuthClientLayer` with the given `secret`. + pub const fn new(secret: JwtSecret) -> Self { + Self { secret } + } +} + +impl Layer for AuthClientLayer { + type Service = AuthClientService; + + fn layer(&self, inner: S) -> Self::Service { + AuthClientService::new(self.secret, inner) + } +} + +/// Automatically authenticates every client request with the given `secret`. +#[derive(Debug, Clone)] +pub struct AuthClientService { + secret: JwtSecret, + inner: S, +} + +impl AuthClientService { + const fn new(secret: JwtSecret, inner: S) -> Self { + Self { secret, inner } + } +} + +impl Service> for AuthClientService +where + S: Service>, + B: std::fmt::Debug, +{ + type Response = S::Response; + type Error = S::Error; + type Future = S::Future; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, mut request: http::Request) -> Self::Future { + request + .headers_mut() + .insert(AUTHORIZATION, secret_to_bearer_header(&self.secret)); + self.inner.call(request) + } +} + +/// Helper function to convert a secret into a Bearer auth header value with claims according to +/// . +/// The token is valid for 60 seconds. +pub fn secret_to_bearer_header(secret: &JwtSecret) -> HeaderValue { + format!( + "Bearer {}", + secret + .encode(&Claims { + iat: (SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + + Duration::from_secs(60)) + .as_secs(), + exp: None, + }) + .unwrap() + ) + .parse() + .unwrap() +} diff --git a/src/client/http.rs b/src/client/http.rs new file mode 100644 index 00000000..ecd390b7 --- /dev/null +++ b/src/client/http.rs @@ -0,0 +1,142 @@ +use crate::client::auth::{AuthClientLayer, AuthClientService}; +use crate::server::PayloadSource; +use alloy_rpc_types_engine::JwtSecret; +use flate2::read::GzDecoder; +use http::response::Parts; +use http::{Request, Uri}; +use http_body_util::BodyExt; +use hyper_rustls::HttpsConnector; +use hyper_util::client::legacy::Client; +use hyper_util::client::legacy::connect::HttpConnector; +use hyper_util::rt::TokioExecutor; +use jsonrpsee::core::BoxError; +use jsonrpsee::http_client::HttpBody; +use opentelemetry::trace::SpanKind; +use std::io::Read; +use tower::{Layer, Service}; +use tracing::{debug, error, instrument, warn}; + +#[derive(Clone, Debug)] +pub(crate) struct HttpClient { + client: AuthClientService, HttpBody>>, + url: Uri, + target: PayloadSource, +} + +impl HttpClient { + pub(crate) fn new(url: Uri, secret: JwtSecret, target: PayloadSource) -> Self { + let connector = hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .expect("no native root CA certificates found") + .https_or_http() + .enable_http1() + .enable_http2() + .build(); + let auth = AuthClientLayer::new(secret); + let client: Client, HttpBody> = + Client::builder(TokioExecutor::new()).build(connector); + let client = auth.layer(client); + Self { + client, + url, + target, + } + } + + #[instrument( + skip(self, req), + fields(otel.kind = ?SpanKind::Client), + err + )] + pub(crate) async fn forward( + &mut self, + req: Request, + method: String, + ) -> Result, BoxError> { + debug!("forwarding {} to {}", method, self.target); + debug!("{:?}", req); + + let res = self.client.call(req).await?; + + let (parts, body) = res.into_parts(); + let body_bytes = body + .collect() + .await + .map_err(|e| { + error!( + target: "proxy::forward_request", + message = "error collecting body", + error = %e, + ); + e + })? + .to_bytes() + .to_vec(); + let parts_clone = parts.clone(); + let body_bytes_clone = body_bytes.clone(); + + self.process_response(parts_clone, body_bytes_clone) + .await + .unwrap(); + + Ok(http::Response::from_parts( + parts, + HttpBody::from(body_bytes), + )) + } + + async fn process_response(&self, parts: Parts, body_bytes: Vec) -> Result<(), BoxError> { + // Check for GZIP compression + let is_gzipped = parts + .headers + .get(http::header::CONTENT_ENCODING) + .is_some_and(|val| val.as_bytes() == b"gzip"); + let decoded_body = if is_gzipped { + // Decompress GZIP content + let mut decoder = GzDecoder::new(&body_bytes[..]); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).map_err(|e| { + warn!( + target: "proxy::process_response", + message = "error decompressing body", + error = %e, + ); + e + })?; + decoded + } else { + body_bytes + }; + + // log the decoded body + debug!( + target: "proxy::forward_request", + message = "raw response body", + body = %String::from_utf8_lossy(&decoded_body), + ); + + let _ = parse_response_code(&decoded_body); + + Ok(()) + } +} + +fn parse_response_code(body_bytes: &[u8]) -> Option { + #[derive(serde::Deserialize, Debug)] + struct RpcResponse { + error: Option, + } + + #[derive(serde::Deserialize, Debug)] + struct JsonRpcError { + code: i32, + } + + // Safely try to deserialize, return empty string on failure + serde_json::from_slice::(body_bytes) + .map_err(|e| { + warn!(target: "proxy::parse_response_code", message = "error deserializing body", error = %e); + }) + .ok() + .and_then(|r| r.error.map(|e| e.code.to_string())) +} diff --git a/src/client/mod.rs b/src/client/mod.rs new file mode 100644 index 00000000..634f28de --- /dev/null +++ b/src/client/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod auth; +pub(crate) mod http; +pub(crate) mod rpc; diff --git a/src/client.rs b/src/client/rpc.rs similarity index 92% rename from src/client.rs rename to src/client/rpc.rs index c28235b2..14f81bab 100644 --- a/src/client.rs +++ b/src/client/rpc.rs @@ -1,4 +1,4 @@ -use crate::auth_layer::{AuthClientLayer, AuthClientService}; +use crate::client::auth::{AuthClientLayer, AuthClientService}; use crate::server::{EngineApiClient, PayloadSource}; use alloy_primitives::B256; use alloy_rpc_types_engine::{ @@ -18,10 +18,10 @@ use std::time::Duration; use thiserror::Error; use tracing::{error, info, instrument}; -pub(crate) type ClientResult = Result; +pub(crate) type ClientResult = Result; #[derive(Error, Debug)] -pub(crate) enum ClientError { +pub(crate) enum RpcClientError { #[error(transparent)] Jsonrpsee(#[from] jsonrpsee::core::client::Error), #[error("Invalid payload: {0}")] @@ -32,10 +32,10 @@ pub(crate) enum ClientError { Jwt(#[from] JwtError), } -impl From for ErrorObjectOwned { - fn from(err: ClientError) -> Self { +impl From for ErrorObjectOwned { + fn from(err: RpcClientError) -> Self { match err { - ClientError::Jsonrpsee(jsonrpsee::core::ClientError::Call(error_object)) => { + RpcClientError::Jsonrpsee(jsonrpsee::core::ClientError::Call(error_object)) => { error_object } // Status code 13 == internal error @@ -49,7 +49,7 @@ impl From for ErrorObjectOwned { /// - **Engine API** calls are faciliated via the `auth_client` (requires JWT authentication). /// #[derive(Clone)] -pub(crate) struct ExecutionClient { +pub(crate) struct RpcClient { /// Handles requests to the authenticated Engine API (requires JWT authentication) auth_client: HttpClient>, /// Uri of the RPC server for authenticated Engine API calls @@ -58,14 +58,14 @@ pub(crate) struct ExecutionClient { payload_source: PayloadSource, } -impl ExecutionClient { +impl RpcClient { /// Initializes a new [ExecutionClient] with JWT auth for the Engine API and without auth for general execution layer APIs. pub fn new( auth_rpc: Uri, auth_rpc_jwt_secret: JwtSecret, timeout: u64, payload_source: PayloadSource, - ) -> Result { + ) -> Result { let auth_layer = AuthClientLayer::new(auth_rpc_jwt_secret); let auth_client = HttpClientBuilder::new() .set_http_middleware(tower::ServiceBuilder::new().layer(auth_layer)) @@ -106,7 +106,7 @@ impl ExecutionClient { } if res.is_invalid() { - return Err(ClientError::InvalidPayload( + return Err(RpcClientError::InvalidPayload( res.payload_status.status.to_string(), )); } @@ -159,7 +159,7 @@ impl ExecutionClient { .await?; if res.is_invalid() { - return Err(ClientError::InvalidPayload(res.status.to_string())); + return Err(RpcClientError::InvalidPayload(res.status.to_string())); } Ok(res) @@ -202,7 +202,7 @@ mod tests { use http::Uri; use jsonrpsee::core::client::ClientT; - use crate::auth_layer::AuthClientService; + use crate::client::auth::AuthClientService; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; use jsonrpsee::RpcModule; @@ -241,7 +241,7 @@ mod tests { let secret = JwtSecret::from_hex(SECRET).unwrap(); let auth_rpc = Uri::from_str(&format!("http://{}:{}", AUTH_ADDR, AUTH_PORT)).unwrap(); - let client = ExecutionClient::new(auth_rpc, secret, 1000, PayloadSource::L2).unwrap(); + let client = RpcClient::new(auth_rpc, secret, 1000, PayloadSource::L2).unwrap(); let response = send_request(client.auth_client).await; assert!(response.is_ok()); assert_eq!(response.unwrap(), "You are the dark lord"); @@ -251,7 +251,7 @@ mod tests { async fn invalid_jwt() { let secret = JwtSecret::random(); let auth_rpc = Uri::from_str(&format!("http://{}:{}", AUTH_ADDR, AUTH_PORT)).unwrap(); - let client = ExecutionClient::new(auth_rpc, secret, 1000, PayloadSource::L2).unwrap(); + let client = RpcClient::new(auth_rpc, secret, 1000, PayloadSource::L2).unwrap(); let response = send_request(client.auth_client).await; assert!(response.is_err()); assert!(matches!( diff --git a/src/integration/mod.rs b/src/integration/mod.rs index 9f688e0a..c326c9c2 100644 --- a/src/integration/mod.rs +++ b/src/integration/mod.rs @@ -1,4 +1,4 @@ -use crate::auth_layer::{AuthClientLayer, AuthClientService}; +use crate::client::auth::{AuthClientLayer, AuthClientService}; use crate::debug_api::DebugClient; use crate::server::EngineApiClient; use crate::server::PayloadSource; diff --git a/src/main.rs b/src/main.rs index 775cf9ed..941bdda0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ #![allow(clippy::complexity)] +use crate::client::rpc::{BuilderArgs, L2ClientArgs, RpcClient}; use ::tracing::{Level, error, info}; use clap::{Parser, Subcommand, arg}; -use client::{BuilderArgs, ExecutionClient, L2ClientArgs}; +use client::auth::AuthClientLayer; use debug_api::DebugClient; use health::HealthLayer; use metrics::init_metrics; @@ -25,7 +26,6 @@ use server::{ExecutionMode, PayloadSource, RollupBoostServer}; use tokio::net::TcpListener; use tokio::signal::unix::{SignalKind, signal as unix_signal}; -mod auth_layer; mod client; mod debug_api; mod health; @@ -184,7 +184,7 @@ async fn main() -> eyre::Result<()> { bail!("Missing L2 Client JWT secret"); }; - let l2_client = ExecutionClient::new( + let l2_client = RpcClient::new( l2_client_args.l2_url.clone(), l2_auth_jwt, l2_client_args.l2_timeout, @@ -200,7 +200,7 @@ async fn main() -> eyre::Result<()> { bail!("Missing Builder JWT secret"); }; - let builder_client = ExecutionClient::new( + let builder_client = RpcClient::new( builder_args.builder_url.clone(), builder_auth_jwt, builder_args.builder_timeout, @@ -227,8 +227,11 @@ async fn main() -> eyre::Result<()> { // Build and start the server info!("Starting server on :{}", args.rpc_port); + // let auth = AuthClientLayer::new(builder_args.builder_jwt_token.unwrap()); + let http_middleware = tower::ServiceBuilder::new() .layer(HealthLayer) + // .layer(auth) .layer(ProxyLayer::new( l2_client_args.l2_url, l2_auth_jwt, diff --git a/src/proxy.rs b/src/proxy.rs index 9c37f8ad..cb62c029 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -1,12 +1,13 @@ -use crate::auth_layer::secret_to_bearer_header; +use crate::RpcClient; +use crate::client::auth::{AuthClientLayer, AuthClientService, secret_to_bearer_header}; +use crate::client::http::HttpClient; use crate::metrics::ServerMetrics; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; use flate2::read::GzDecoder; -use futures_util::future::FutureExt; use http::header::AUTHORIZATION; use http::response::Parts; -use http::{StatusCode, Uri}; +use http::{Request, Uri}; use http_body_util::BodyExt; use hyper_rustls::HttpsConnector; use hyper_util::client::legacy::Client; @@ -16,7 +17,6 @@ use jsonrpsee::MethodResponse; use jsonrpsee::core::{BoxError, http_helpers}; use jsonrpsee::http_client::{HttpBody, HttpRequest, HttpResponse}; use jsonrpsee::server::middleware::rpc::RpcServiceT; -use jsonrpsee::types::Request; use opentelemetry::trace::SpanKind; use std::io::Read; use std::sync::Arc; @@ -72,22 +72,24 @@ impl Layer for ProxyLayer { type Service = ProxyService; fn layer(&self, inner: S) -> Self::Service { - let connector = hyper_rustls::HttpsConnectorBuilder::new() - .with_native_roots() - .expect("no native root CA certificates found") - .https_or_http() - .enable_http1() - .enable_http2() - .build(); + let l2_client = HttpClient::new( + self.l2_auth_uri.clone(), + self.l2_auth_secret.clone(), + PayloadSource::L2, + ); + + let builder_client = HttpClient::new( + self.builder_auth_uri.clone(), + self.builder_auth_secret.clone(), + PayloadSource::Builder, + ); + + let auth_layer = AuthClientLayer::new(self.l2_auth_secret.clone()); ProxyService { inner, - client: Client::builder(TokioExecutor::new()).build(connector), - l2_auth_uri: self.l2_auth_uri.clone(), - l2_auth_secret: self.l2_auth_secret, - builder_auth_uri: self.builder_auth_uri.clone(), - builder_auth_secret: self.builder_auth_secret, - metrics: self.metrics.clone(), + l2_client, + builder_client, } } } @@ -95,14 +97,11 @@ impl Layer for ProxyLayer { #[derive(Clone)] pub struct ProxyService { inner: S, - client: Client, HttpBody>, - l2_auth_uri: Uri, - l2_auth_secret: JwtSecret, - builder_auth_uri: Uri, - builder_auth_secret: JwtSecret, - metrics: Option>, + l2_client: HttpClient, + builder_client: HttpClient, } +// Consider using `RpcServiceT` when https://github.com/paritytech/jsonrpsee/pull/1521 is merged impl Service> for ProxyService where S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, @@ -120,20 +119,13 @@ where } fn call(&mut self, req: HttpRequest) -> Self::Future { - let client = self.client.clone(); - let mut inner = self.inner.clone(); - let builder_uri = self.builder_auth_uri.clone(); - let builder_secret = self.builder_auth_secret; - let l2_uri = self.l2_auth_uri.clone(); - let l2_secret = self.l2_auth_secret; - let metrics = self.metrics.clone(); - #[derive(serde::Deserialize, Debug)] struct RpcRequest<'a> { #[serde(borrow)] method: &'a str, } + let mut service = self.clone(); let fut = async move { let (parts, body) = req.into_parts(); let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; @@ -145,219 +137,31 @@ where if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { if FORWARD_REQUESTS.contains(&method.as_str()) { - let builder_client = client.clone(); let builder_req = HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); let builder_method = method.clone(); - let builder_metrics = metrics.clone(); + let mut builder_client = service.builder_client.clone(); tokio::spawn(async move { - let _ = forward_request( - builder_client, - builder_req, - builder_method, - builder_uri, - builder_secret, - builder_metrics, - PayloadSource::Builder, - ) - .await; + let _ = builder_client.forward(builder_req, builder_method).await; }); let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - forward_request( - client, - l2_req, - method, - l2_uri, - l2_secret, - metrics, - PayloadSource::L2, - ) - .await + service.l2_client.forward(l2_req, method).await } else { let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - inner.call(req).await.map_err(|e| e.into()) + service.inner.call(req).await.map_err(|e| e.into()) } } else { let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - forward_request( - client, - req, - method, - l2_uri, - l2_secret, - metrics, - PayloadSource::L2, - ) - .await + service.l2_client.forward(req, method).await } }; Box::pin(fut) } } -/// Forwards an HTTP request to the `authrpc``, attaching the provided JWT authorization. -#[instrument( - skip(client, req, auth, metrics), - fields(otel.kind = ?SpanKind::Client), - err -)] -async fn forward_request( - client: Client, HttpBody>, - mut req: http::Request, - method: String, - url: Uri, - auth: JwtSecret, - metrics: Option>, - target: PayloadSource, -) -> Result, BoxError> { - debug!("forwarding {} to {}", method, target); - let start = Instant::now(); - - *req.uri_mut() = url.clone(); - req.headers_mut() - .insert(AUTHORIZATION, secret_to_bearer_header(&auth)); - - let res = client.request(req).await?; - - let (parts, body) = res.into_parts(); - let body_bytes = body - .collect() - .await - .map_err(|e| { - error!( - target: "proxy::forward_request", - message = "error collecting body", - error = %e, - ); - e - })? - .to_bytes() - .to_vec(); - let parts_clone = parts.clone(); - let body_bytes_clone = body_bytes.clone(); - - tokio::spawn(async move { - let _ = process_response( - parts_clone, - body_bytes_clone, - metrics, - method, - start.elapsed(), - target, - ) - .await; - }); - - Ok(http::Response::from_parts( - parts, - HttpBody::from(body_bytes), - )) -} - -async fn process_response( - parts: Parts, - body_bytes: Vec, - metrics: Option>, - method: String, - duration: Duration, - source: PayloadSource, -) -> Result<(), BoxError> { - // Check for GZIP compression - let is_gzipped = parts - .headers - .get(http::header::CONTENT_ENCODING) - .is_some_and(|val| val.as_bytes() == b"gzip"); - let decoded_body = if is_gzipped { - // Decompress GZIP content - let mut decoder = GzDecoder::new(&body_bytes[..]); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).map_err(|e| { - warn!( - target: "proxy::process_response", - message = "error decompressing body", - error = %e, - ); - e - })?; - decoded - } else { - body_bytes - }; - - // log the decoded body - debug!( - target: "proxy::forward_request", - message = "raw response body", - body = %String::from_utf8_lossy(&decoded_body), - ); - - let rpc_status_code = parse_response_code(&decoded_body); - record_metrics( - metrics, - parts.status.to_string(), - rpc_status_code, - method, - duration, - source, - ) - .await; - - Ok(()) -} - -async fn record_metrics( - metrics: Option>, - http_status_code: String, - rpc_status_code: Option, - method: String, - duration: Duration, - source: PayloadSource, -) { - if let Some(metrics) = &metrics { - match source { - PayloadSource::Builder => { - metrics.record_builder_forwarded_call(duration, method.to_string()); - metrics.increment_builder_rpc_response_count( - http_status_code, - rpc_status_code, - method.to_string(), - ); - } - PayloadSource::L2 => { - metrics.record_l2_forwarded_call(duration, method.to_string()); - metrics.increment_l2_rpc_response_count( - http_status_code, - rpc_status_code, - method.to_string(), - ); - } - } - } -} - -fn parse_response_code(body_bytes: &[u8]) -> Option { - #[derive(serde::Deserialize, Debug)] - struct RpcResponse { - error: Option, - } - - #[derive(serde::Deserialize, Debug)] - struct JsonRpcError { - code: i32, - } - - // Safely try to deserialize, return empty string on failure - serde_json::from_slice::(body_bytes) - .map_err(|e| { - warn!(target: "proxy::parse_response_code", message = "error deserializing body", error = %e); - }) - .ok() - .and_then(|r| r.error.map(|e| e.code.to_string())) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/server.rs b/src/server.rs index 656940b4..36bb3381 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,4 +1,4 @@ -use crate::client::ExecutionClient; +use crate::client::rpc::RpcClient; use crate::debug_api::DebugServer; use alloy_primitives::B256; use moka::sync::Cache; @@ -114,8 +114,8 @@ impl ExecutionMode { #[derive(Clone)] pub struct RollupBoostServer { - pub l2_client: Arc, - pub builder_client: Arc, + pub l2_client: Arc, + pub builder_client: Arc, pub boost_sync: bool, pub payload_trace_context: Arc, execution_mode: Arc>, @@ -123,8 +123,8 @@ pub struct RollupBoostServer { impl RollupBoostServer { pub fn new( - l2_client: ExecutionClient, - builder_client: ExecutionClient, + l2_client: RpcClient, + builder_client: RpcClient, boost_sync: bool, initial_execution_mode: ExecutionMode, ) -> Self { @@ -522,13 +522,12 @@ mod tests { let l2_auth_rpc = Uri::from_str(&format!("http://{}:{}", HOST, L2_PORT)).unwrap(); let l2_client = - ExecutionClient::new(l2_auth_rpc, jwt_secret, 2000, PayloadSource::L2).unwrap(); + RpcClient::new(l2_auth_rpc, jwt_secret, 2000, PayloadSource::L2).unwrap(); let builder_auth_rpc = Uri::from_str(&format!("http://{}:{}", HOST, BUILDER_PORT)).unwrap(); let builder_client = - ExecutionClient::new(builder_auth_rpc, jwt_secret, 2000, PayloadSource::Builder) - .unwrap(); + RpcClient::new(builder_auth_rpc, jwt_secret, 2000, PayloadSource::Builder).unwrap(); let rollup_boost_client = RollupBoostServer::new( l2_client, From 6249337965a7f792316ce1a9d7565b28b255b557 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 12:06:51 -0700 Subject: [PATCH 05/39] clean things up --- src/client/http.rs | 4 +- src/health.rs | 15 +++--- src/main.rs | 67 ++++---------------------- src/metrics.rs | 116 +++++++++++++++++++-------------------------- src/proxy.rs | 60 +++++++++-------------- 5 files changed, 90 insertions(+), 172 deletions(-) diff --git a/src/client/http.rs b/src/client/http.rs index ecd390b7..56e7e2b8 100644 --- a/src/client/http.rs +++ b/src/client/http.rs @@ -50,11 +50,11 @@ impl HttpClient { )] pub(crate) async fn forward( &mut self, - req: Request, + mut req: Request, method: String, ) -> Result, BoxError> { + *req.uri_mut() = self.url.clone(); debug!("forwarding {} to {}", method, self.target); - debug!("{:?}", req); let res = self.client.call(req).await?; diff --git a/src/health.rs b/src/health.rs index 9592ed6f..36bfbb62 100644 --- a/src/health.rs +++ b/src/health.rs @@ -11,18 +11,21 @@ use jsonrpsee::{ use tower::{Layer, Service, util::Either}; /// A [`Layer`] that filters out /healthz requests and responds with a 200 OK. +#[derive(Clone, Debug)] pub(crate) struct HealthLayer; impl Layer for HealthLayer { type Service = HealthService; - fn layer(&self, service: S) -> Self::Service { - HealthService(service) + fn layer(&self, inner: S) -> Self::Service { + HealthService { inner } } } -#[derive(Clone)] -pub struct HealthService(S); +#[derive(Clone, Debug)] +pub struct HealthService { + inner: S, +} impl Service> for HealthService where @@ -39,14 +42,14 @@ where >; fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.0.poll_ready(cx).map_err(Into::into) + self.inner.poll_ready(cx).map_err(Into::into) } fn call(&mut self, request: HttpRequest) -> Self::Future { if request.uri().path() == "/healthz" { Either::A(Self::healthz().boxed()) } else { - Either::B(self.0.call(request)) + Either::B(self.inner.call(request)) } } } diff --git a/src/main.rs b/src/main.rs index 941bdda0..17c13c36 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,7 @@ #![allow(clippy::complexity)] use crate::client::rpc::{BuilderArgs, L2ClientArgs, RpcClient}; -use ::tracing::{Level, error, info}; +use ::tracing::{Level, info}; use clap::{Parser, Subcommand, arg}; -use client::auth::AuthClientLayer; use debug_api::DebugClient; use health::HealthLayer; use metrics::init_metrics; @@ -12,18 +11,11 @@ use tracing::init_tracing; use alloy_rpc_types_engine::JwtSecret; use dotenv::dotenv; use eyre::bail; -use http::StatusCode; -use hyper::service::service_fn; -use hyper::{Request, Response, server::conn::http1}; -use hyper_util::rt::TokioIo; use jsonrpsee::RpcModule; -use jsonrpsee::http_client::HttpBody; use jsonrpsee::server::Server; -use metrics_exporter_prometheus::PrometheusHandle; use proxy::ProxyLayer; use server::{ExecutionMode, PayloadSource, RollupBoostServer}; -use tokio::net::TcpListener; use tokio::signal::unix::{SignalKind, signal as unix_signal}; mod client; @@ -172,7 +164,7 @@ async fn main() -> eyre::Result<()> { } init_tracing(&args)?; - let metrics = init_metrics(&args)?; + init_metrics(&args)?; let l2_client_args = args.l2_client; @@ -227,18 +219,12 @@ async fn main() -> eyre::Result<()> { // Build and start the server info!("Starting server on :{}", args.rpc_port); - // let auth = AuthClientLayer::new(builder_args.builder_jwt_token.unwrap()); - - let http_middleware = tower::ServiceBuilder::new() - .layer(HealthLayer) - // .layer(auth) - .layer(ProxyLayer::new( - l2_client_args.l2_url, - l2_auth_jwt, - builder_args.builder_url, - builder_auth_jwt, - metrics, - )); + let http_middleware = tower::ServiceBuilder::new().layer(ProxyLayer::new( + l2_client_args.l2_url, + l2_auth_jwt, + builder_args.builder_url, + builder_auth_jwt, + )); let server = Server::builder() .set_http_middleware(http_middleware) @@ -269,40 +255,3 @@ async fn main() -> eyre::Result<()> { Ok(()) } - -async fn init_metrics_server(addr: SocketAddr, handle: PrometheusHandle) -> eyre::Result<()> { - let listener = TcpListener::bind(addr).await?; - info!("Metrics server running on {}", addr); - - loop { - match listener.accept().await { - Ok((stream, _)) => { - let handle = handle.clone(); // Clone the handle for each connection - tokio::task::spawn(async move { - let service = service_fn(move |_req: Request| { - let response = match _req.uri().path() { - "/metrics" => Response::builder() - .header("content-type", "text/plain") - .body(HttpBody::from(handle.render())) - .unwrap(), - _ => Response::builder() - .status(StatusCode::NOT_FOUND) - .body(HttpBody::empty()) - .unwrap(), - }; - async { Ok::<_, hyper::Error>(response) } - }); - - let io = TokioIo::new(stream); - - if let Err(err) = http1::Builder::new().serve_connection(io, service).await { - error!(message = "Error serving metrics connection", error = %err); - } - }); - } - Err(e) => { - error!(message = "Error accepting connection", error = %e); - } - } - } -} diff --git a/src/metrics.rs b/src/metrics.rs index 383c2ba9..2135a885 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1,74 +1,21 @@ -use std::{net::SocketAddr, sync::Arc, time::Duration}; +use std::net::SocketAddr; use eyre::Result; -use metrics::{Counter, Histogram, counter, histogram}; -use metrics_derive::Metrics; use metrics_exporter_prometheus::PrometheusBuilder; use metrics_util::layers::{PrefixLayer, Stack}; +use tokio::net::TcpListener; +use tracing::{error, info}; -use crate::{Args, init_metrics_server}; +use crate::Args; -#[derive(Metrics)] -#[metrics(scope = "rpc")] -pub struct ServerMetrics { - // Builder proxy metrics - #[metric(describe = "Latency for builder client forwarded rpc calls (excluding the engine api)", labels = ["method"])] - #[allow(dead_code)] - pub builder_forwarded_call: Histogram, +use http::StatusCode; +use hyper::service::service_fn; +use hyper::{Request, Response, server::conn::http1}; +use hyper_util::rt::TokioIo; +use jsonrpsee::http_client::HttpBody; +use metrics_exporter_prometheus::PrometheusHandle; - #[metric(describe = "Number of builder client rpc responses", labels = ["http_status_code", "rpc_status_code", "method"])] - #[allow(dead_code)] - pub builder_rpc_response_count: Counter, - - // L2 proxy metrics - #[metric(describe = "Latency for l2 client forwarded rpc calls (excluding the engine api)", labels = ["method"])] - #[allow(dead_code)] - pub l2_forwarded_call: Histogram, - - #[metric(describe = "Number of l2 client rpc responses", labels = ["http_status_code", "rpc_status_code", "method"])] - #[allow(dead_code)] - pub l2_rpc_response_count: Counter, -} - -impl ServerMetrics { - pub fn record_builder_forwarded_call(&self, latency: Duration, method: String) { - histogram!("rpc.builder_forwarded_call", "method" => method).record(latency.as_secs_f64()); - } - - pub fn increment_builder_rpc_response_count( - &self, - http_status_code: String, - rpc_status_code: Option, - method: String, - ) { - counter!("rpc.builder_response_count", - "http_status_code" => http_status_code, - "rpc_status_code" => rpc_status_code.unwrap_or("".to_string()), - "method" => method, - ) - .increment(1); - } - - pub fn record_l2_forwarded_call(&self, latency: Duration, method: String) { - histogram!("rpc.l2_forwarded_call", "method" => method).record(latency.as_secs_f64()); - } - - pub fn increment_l2_rpc_response_count( - &self, - http_status_code: String, - rpc_status_code: Option, - method: String, - ) { - let mut labels = vec![("http_status_code", http_status_code), ("method", method)]; - if let Some(rpc_status_code) = rpc_status_code { - labels.push(("rpc_status_code", rpc_status_code)); - } - - counter!("rpc.l2_response_count", &labels).increment(1); - } -} - -pub(crate) fn init_metrics(args: &Args) -> Result>> { +pub(crate) fn init_metrics(args: &Args) -> Result<()> { if args.metrics { let recorder = PrometheusBuilder::new().build_recorder(); let handle = recorder.handle(); @@ -81,8 +28,43 @@ pub(crate) fn init_metrics(args: &Args) -> Result>> { let metrics_addr = format!("{}:{}", args.metrics_host, args.metrics_port); let addr: SocketAddr = metrics_addr.parse()?; tokio::spawn(init_metrics_server(addr, handle)); // Run the metrics server in a separate task - Ok(Some(Arc::new(ServerMetrics::default()))) - } else { - Ok(None) + } + Ok(()) +} + +async fn init_metrics_server(addr: SocketAddr, handle: PrometheusHandle) -> eyre::Result<()> { + let listener = TcpListener::bind(addr).await?; + info!("Metrics server running on {}", addr); + + loop { + match listener.accept().await { + Ok((stream, _)) => { + let handle = handle.clone(); // Clone the handle for each connection + tokio::task::spawn(async move { + let service = service_fn(move |_req: Request| { + let response = match _req.uri().path() { + "/metrics" => Response::builder() + .header("content-type", "text/plain") + .body(HttpBody::from(handle.render())) + .unwrap(), + _ => Response::builder() + .status(StatusCode::NOT_FOUND) + .body(HttpBody::empty()) + .unwrap(), + }; + async { Ok::<_, hyper::Error>(response) } + }); + + let io = TokioIo::new(stream); + + if let Err(err) = http1::Builder::new().serve_connection(io, service).await { + error!(message = "Error serving metrics connection", error = %err); + } + }); + } + Err(e) => { + error!(message = "Error accepting connection", error = %e); + } + } } } diff --git a/src/proxy.rs b/src/proxy.rs index cb62c029..498a02f5 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -1,30 +1,15 @@ -use crate::RpcClient; -use crate::client::auth::{AuthClientLayer, AuthClientService, secret_to_bearer_header}; +use crate::HealthLayer; use crate::client::http::HttpClient; -use crate::metrics::ServerMetrics; +use crate::health::HealthService; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; -use flate2::read::GzDecoder; -use http::header::AUTHORIZATION; -use http::response::Parts; -use http::{Request, Uri}; -use http_body_util::BodyExt; -use hyper_rustls::HttpsConnector; -use hyper_util::client::legacy::Client; -use hyper_util::client::legacy::connect::HttpConnector; -use hyper_util::rt::TokioExecutor; -use jsonrpsee::MethodResponse; +use http::Uri; use jsonrpsee::core::{BoxError, http_helpers}; use jsonrpsee::http_client::{HttpBody, HttpRequest, HttpResponse}; -use jsonrpsee::server::middleware::rpc::RpcServiceT; -use opentelemetry::trace::SpanKind; -use std::io::Read; -use std::sync::Arc; use std::task::{Context, Poll}; -use std::time::{Duration, Instant}; use std::{future::Future, pin::Pin}; use tower::{Layer, Service}; -use tracing::{debug, error, info, instrument, warn}; +use tracing::info; const MULTIPLEX_METHODS: [&str; 4] = [ "engine_", @@ -43,54 +28,51 @@ const FORWARD_REQUESTS: [&str; 6] = [ #[derive(Debug, Clone)] pub struct ProxyLayer { - l2_auth_uri: Uri, + l2_auth_rpc: Uri, l2_auth_secret: JwtSecret, - builder_auth_uri: Uri, + builder_auth_rpc: Uri, builder_auth_secret: JwtSecret, - metrics: Option>, } impl ProxyLayer { pub fn new( - l2_auth_uri: Uri, + l2_auth_rpc: Uri, l2_auth_secret: JwtSecret, - builder_auth_uri: Uri, + builder_auth_rpc: Uri, builder_auth_secret: JwtSecret, - metrics: Option>, ) -> Self { ProxyLayer { - l2_auth_uri, + l2_auth_rpc, l2_auth_secret, - builder_auth_uri, + builder_auth_rpc, builder_auth_secret, - metrics, } } } impl Layer for ProxyLayer { - type Service = ProxyService; + type Service = HealthService>; fn layer(&self, inner: S) -> Self::Service { let l2_client = HttpClient::new( - self.l2_auth_uri.clone(), + self.l2_auth_rpc.clone(), self.l2_auth_secret.clone(), PayloadSource::L2, ); let builder_client = HttpClient::new( - self.builder_auth_uri.clone(), + self.builder_auth_rpc.clone(), self.builder_auth_secret.clone(), PayloadSource::Builder, ); - let auth_layer = AuthClientLayer::new(self.l2_auth_secret.clone()); - - ProxyService { + let proxy = ProxyService { inner, l2_client, builder_client, - } + }; + + HealthLayer.layer(proxy) } } @@ -170,7 +152,9 @@ mod tests { use alloy_rpc_types_eth::erc4337::TransactionConditional; use http_body_util::BodyExt; use hyper::service::service_fn; - use hyper_util::rt::TokioIo; + use hyper_util::client::legacy::Client; + use hyper_util::client::legacy::connect::HttpConnector; + use hyper_util::rt::{TokioExecutor, TokioIo}; use jsonrpsee::server::Server; use jsonrpsee::{ RpcModule, @@ -215,7 +199,7 @@ mod tests { JwtSecret::random(), format!("http://{}:{}", builder.addr.ip(), builder.addr.port()).parse::()?, JwtSecret::random(), - None, + // None, )); let temp_listener = TcpListener::bind("0.0.0.0:0").await?; @@ -477,7 +461,7 @@ mod tests { .parse::() .unwrap(); - let proxy_layer = ProxyLayer::new(l2_auth_uri.clone(), jwt, l2_auth_uri, jwt, None); + let proxy_layer = ProxyLayer::new(l2_auth_uri.clone(), jwt, l2_auth_uri, jwt); // Create a layered server let server = ServerBuilder::default() From 0641965c39626dea3a8e60501388c2b87c4356fe Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 12:41:38 -0700 Subject: [PATCH 06/39] fix for cloned service --- src/proxy.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/proxy.rs b/src/proxy.rs index 498a02f5..977bab4e 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -76,7 +76,7 @@ impl Layer for ProxyLayer { } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct ProxyService { inner: S, l2_client: HttpClient, @@ -107,7 +107,11 @@ where method: &'a str, } + // See https://github.com/tower-rs/tower/blob/abb375d08cf0ba34c1fe76f66f1aba3dc4341013/tower-service/src/lib.rs#L276 + // for an explanation of this pattern let mut service = self.clone(); + service.inner = std::mem::replace(&mut self.inner, service.inner); + let fut = async move { let (parts, body) = req.into_parts(); let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; From 79bab4aa1243d96b49237a0592ffd0cf2e02c88a Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 13:05:09 -0700 Subject: [PATCH 07/39] cleanup process_response --- src/client/http.rs | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/src/client/http.rs b/src/client/http.rs index 56e7e2b8..ca210873 100644 --- a/src/client/http.rs +++ b/src/client/http.rs @@ -1,6 +1,7 @@ use crate::client::auth::{AuthClientLayer, AuthClientService}; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; +use eyre::Context; use flate2::read::GzDecoder; use http::response::Parts; use http::{Request, Uri}; @@ -14,7 +15,7 @@ use jsonrpsee::http_client::HttpBody; use opentelemetry::trace::SpanKind; use std::io::Read; use tower::{Layer, Service}; -use tracing::{debug, error, instrument, warn}; +use tracing::{debug, error, instrument}; #[derive(Clone, Debug)] pub(crate) struct HttpClient { @@ -75,9 +76,8 @@ impl HttpClient { let parts_clone = parts.clone(); let body_bytes_clone = body_bytes.clone(); - self.process_response(parts_clone, body_bytes_clone) - .await - .unwrap(); + self.handle_response(parts_clone, body_bytes_clone) + .context("error fowarding request")?; Ok(http::Response::from_parts( parts, @@ -85,43 +85,38 @@ impl HttpClient { )) } - async fn process_response(&self, parts: Parts, body_bytes: Vec) -> Result<(), BoxError> { + fn handle_response(&self, parts: Parts, body_bytes: Vec) -> eyre::Result<()> { // Check for GZIP compression let is_gzipped = parts .headers .get(http::header::CONTENT_ENCODING) .is_some_and(|val| val.as_bytes() == b"gzip"); + let decoded_body = if is_gzipped { // Decompress GZIP content let mut decoder = GzDecoder::new(&body_bytes[..]); let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).map_err(|e| { - warn!( - target: "proxy::process_response", - message = "error decompressing body", - error = %e, - ); - e - })?; + decoder + .read_to_end(&mut decoded) + .context("error decompressing body")?; decoded } else { body_bytes }; - // log the decoded body debug!( - target: "proxy::forward_request", - message = "raw response body", + target: "proxy::forward_request", + message = "raw response body", body = %String::from_utf8_lossy(&decoded_body), ); - let _ = parse_response_code(&decoded_body); + handle_response_code(&decoded_body)?; Ok(()) } } -fn parse_response_code(body_bytes: &[u8]) -> Option { +fn handle_response_code(body_bytes: &[u8]) -> eyre::Result<()> { #[derive(serde::Deserialize, Debug)] struct RpcResponse { error: Option, @@ -132,11 +127,12 @@ fn parse_response_code(body_bytes: &[u8]) -> Option { code: i32, } - // Safely try to deserialize, return empty string on failure - serde_json::from_slice::(body_bytes) - .map_err(|e| { - warn!(target: "proxy::parse_response_code", message = "error deserializing body", error = %e); - }) - .ok() - .and_then(|r| r.error.map(|e| e.code.to_string())) + let res = + serde_json::from_slice::(body_bytes).context("error deserializing body")?; + + if let Some(e) = res.error { + return Err(eyre::eyre!("code: {}", e.code)); + } + + Ok(()) } From 8ceabac0616efaad0e0ecced13dc9da8224ef1de Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 13:08:06 -0700 Subject: [PATCH 08/39] eyre bail --- src/client/http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/http.rs b/src/client/http.rs index ca210873..5de2a3c1 100644 --- a/src/client/http.rs +++ b/src/client/http.rs @@ -131,7 +131,7 @@ fn handle_response_code(body_bytes: &[u8]) -> eyre::Result<()> { serde_json::from_slice::(body_bytes).context("error deserializing body")?; if let Some(e) = res.error { - return Err(eyre::eyre!("code: {}", e.code)); + eyre::bail!("code: {}", e.code) } Ok(()) From bf55d6ec37d5ead73ef4ae44bef5c65068103c75 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 13:34:49 -0700 Subject: [PATCH 09/39] remove unnecessary deps --- Cargo.toml | 2 +- src/client/mod.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index efc3a93b..85c66977 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ moka = { version = "0.12.10", features = ["sync"] } reqwest = "0.12.5" http = "1.1.0" dotenv = "0.15.0" -tower = { version = "0.4.13", features = ["filter"] } +tower = "0.4.13" http-body = "1.0.1" http-body-util = "0.1.2" hyper = { version = "1.4.1", features = ["full"] } diff --git a/src/client/mod.rs b/src/client/mod.rs index 634f28de..364eb9b9 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,3 +1,3 @@ -pub(crate) mod auth; -pub(crate) mod http; -pub(crate) mod rpc; +pub mod auth; +pub mod http; +pub mod rpc; From b9b242e3e0f60e5cabccdc793d950c96c1dd4a5e Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 18:34:02 -0700 Subject: [PATCH 10/39] Add kubernetes probe layer --- Cargo.lock | 19 ++++- Cargo.toml | 2 + src/client/auth.rs | 21 +++--- src/client/http.rs | 10 ++- src/client/rpc.rs | 65 +++++++++-------- src/health.rs | 67 ------------------ src/integration/mod.rs | 10 +-- src/main.rs | 25 ++++--- src/probe.rs | 155 +++++++++++++++++++++++++++++++++++++++++ src/proxy.rs | 101 +++++++++++++++------------ src/server.rs | 16 ++++- 11 files changed, 321 insertions(+), 170 deletions(-) delete mode 100644 src/health.rs create mode 100644 src/probe.rs diff --git a/Cargo.lock b/Cargo.lock index 8c5b0592..0ba66e23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3755,7 +3755,7 @@ dependencies = [ "jsonrpsee-http-client", "pin-project", "tower 0.4.13", - "tower-http", + "tower-http 0.6.2", "tracing", ] @@ -3855,6 +3855,7 @@ dependencies = [ "tokio", "tokio-util", "tower 0.4.13", + "tower-http 0.5.2", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -4856,6 +4857,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.9.0", + "bytes", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.6.2" diff --git a/Cargo.toml b/Cargo.toml index 85c66977..40143552 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ reqwest = "0.12.5" http = "1.1.0" dotenv = "0.15.0" tower = "0.4.13" +tower-http = { version = "0.5.2", features = ["sensitive-headers"] } http-body = "1.0.1" http-body-util = "0.1.2" hyper = { version = "1.4.1", features = ["full"] } @@ -68,4 +69,5 @@ reth-rpc-layer = { git = "https://github.com/paradigmxyz/reth.git", rev = "v1.3. ctor = "0.4.1" [features] +default = ["integration"] integration = [] diff --git a/src/client/auth.rs b/src/client/auth.rs index 86723fc4..4446fd60 100644 --- a/src/client/auth.rs +++ b/src/client/auth.rs @@ -2,46 +2,51 @@ use alloy_rpc_types_engine::{Claims, JwtSecret}; use http::{HeaderValue, header::AUTHORIZATION}; use std::{ + iter::once, task::{Context, Poll}, time::{Duration, SystemTime, UNIX_EPOCH}, }; use tower::{Layer, Service}; +use tower_http::sensitive_headers::{SetSensitiveRequestHeaders, SetSensitiveRequestHeadersLayer}; + +pub type Auth = AuthService>; /// A layer that adds a new JWT token to every request using `AuthClientService`. #[derive(Clone, Debug)] -pub struct AuthClientLayer { +pub struct AuthLayer { secret: JwtSecret, } -impl AuthClientLayer { +impl AuthLayer { /// Create a new `AuthClientLayer` with the given `secret`. pub const fn new(secret: JwtSecret) -> Self { Self { secret } } } -impl Layer for AuthClientLayer { - type Service = AuthClientService; +impl Layer for AuthLayer { + type Service = AuthService>; fn layer(&self, inner: S) -> Self::Service { - AuthClientService::new(self.secret, inner) + let inner = SetSensitiveRequestHeadersLayer::new(once(AUTHORIZATION)).layer(inner); + AuthService::new(self.secret, inner) } } /// Automatically authenticates every client request with the given `secret`. #[derive(Debug, Clone)] -pub struct AuthClientService { +pub struct AuthService { secret: JwtSecret, inner: S, } -impl AuthClientService { +impl AuthService { const fn new(secret: JwtSecret, inner: S) -> Self { Self { secret, inner } } } -impl Service> for AuthClientService +impl Service> for AuthService where S: Service>, B: std::fmt::Debug, diff --git a/src/client/http.rs b/src/client/http.rs index 5de2a3c1..cce67307 100644 --- a/src/client/http.rs +++ b/src/client/http.rs @@ -1,4 +1,4 @@ -use crate::client::auth::{AuthClientLayer, AuthClientService}; +use crate::client::auth::AuthLayer; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; use eyre::Context; @@ -17,9 +17,13 @@ use std::io::Read; use tower::{Layer, Service}; use tracing::{debug, error, instrument}; +use super::auth::Auth; + +pub type HttpClientService = Auth, HttpBody>>; + #[derive(Clone, Debug)] pub(crate) struct HttpClient { - client: AuthClientService, HttpBody>>, + client: HttpClientService, url: Uri, target: PayloadSource, } @@ -33,7 +37,7 @@ impl HttpClient { .enable_http1() .enable_http2() .build(); - let auth = AuthClientLayer::new(secret); + let auth = AuthLayer::new(secret); let client: Client, HttpBody> = Client::builder(TokioExecutor::new()).build(connector); let client = auth.layer(client); diff --git a/src/client/rpc.rs b/src/client/rpc.rs index 14f81bab..e12c4423 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -1,4 +1,4 @@ -use crate::client::auth::{AuthClientLayer, AuthClientService}; +use crate::client::auth::AuthLayer; use crate::server::{EngineApiClient, PayloadSource}; use alloy_primitives::B256; use alloy_rpc_types_engine::{ @@ -18,31 +18,9 @@ use std::time::Duration; use thiserror::Error; use tracing::{error, info, instrument}; -pub(crate) type ClientResult = Result; +use super::auth::Auth; -#[derive(Error, Debug)] -pub(crate) enum RpcClientError { - #[error(transparent)] - Jsonrpsee(#[from] jsonrpsee::core::client::Error), - #[error("Invalid payload: {0}")] - InvalidPayload(String), - #[error(transparent)] - Io(#[from] std::io::Error), - #[error(transparent)] - Jwt(#[from] JwtError), -} - -impl From for ErrorObjectOwned { - fn from(err: RpcClientError) -> Self { - match err { - RpcClientError::Jsonrpsee(jsonrpsee::core::ClientError::Call(error_object)) => { - error_object - } - // Status code 13 == internal error - e => ErrorObjectOwned::owned(13, e.to_string(), Option::<()>::None), - } - } -} +pub type RpcClientService = HttpClient>; /// Client interface for interacting with execution layer node's Engine API. /// @@ -51,7 +29,7 @@ impl From for ErrorObjectOwned { #[derive(Clone)] pub(crate) struct RpcClient { /// Handles requests to the authenticated Engine API (requires JWT authentication) - auth_client: HttpClient>, + auth_client: RpcClientService, /// Uri of the RPC server for authenticated Engine API calls auth_rpc: Uri, /// The source of the payload @@ -66,7 +44,7 @@ impl RpcClient { timeout: u64, payload_source: PayloadSource, ) -> Result { - let auth_layer = AuthClientLayer::new(auth_rpc_jwt_secret); + let auth_layer = AuthLayer::new(auth_rpc_jwt_secret); let auth_client = HttpClientBuilder::new() .set_http_middleware(tower::ServiceBuilder::new().layer(auth_layer)) .request_timeout(Duration::from_millis(timeout)) @@ -202,13 +180,10 @@ mod tests { use http::Uri; use jsonrpsee::core::client::ClientT; - use crate::client::auth::AuthClientService; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; use jsonrpsee::RpcModule; - use jsonrpsee::http_client::HttpClient; use jsonrpsee::http_client::transport::Error as TransportError; - use jsonrpsee::http_client::transport::HttpBackend; use jsonrpsee::{ core::ClientError, rpc_params, @@ -261,9 +236,7 @@ mod tests { )); } - async fn send_request( - client: HttpClient>, - ) -> Result { + async fn send_request(client: RpcClientService) -> Result { let server = spawn_server().await; let response = client @@ -300,3 +273,29 @@ mod tests { server.start(module) } } + +pub(crate) type ClientResult = Result; + +#[derive(Error, Debug)] +pub(crate) enum RpcClientError { + #[error(transparent)] + Jsonrpsee(#[from] jsonrpsee::core::client::Error), + #[error("Invalid payload: {0}")] + InvalidPayload(String), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Jwt(#[from] JwtError), +} + +impl From for ErrorObjectOwned { + fn from(err: RpcClientError) -> Self { + match err { + RpcClientError::Jsonrpsee(jsonrpsee::core::ClientError::Call(error_object)) => { + error_object + } + // Status code 13 == internal error + e => ErrorObjectOwned::owned(13, e.to_string(), Option::<()>::None), + } + } +} diff --git a/src/health.rs b/src/health.rs deleted file mode 100644 index 36bfbb62..00000000 --- a/src/health.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::{ - pin::Pin, - task::{Context, Poll}, -}; - -use futures::FutureExt as _; -use jsonrpsee::{ - core::BoxError, - http_client::{HttpBody, HttpRequest, HttpResponse}, -}; -use tower::{Layer, Service, util::Either}; - -/// A [`Layer`] that filters out /healthz requests and responds with a 200 OK. -#[derive(Clone, Debug)] -pub(crate) struct HealthLayer; - -impl Layer for HealthLayer { - type Service = HealthService; - - fn layer(&self, inner: S) -> Self::Service { - HealthService { inner } - } -} - -#[derive(Clone, Debug)] -pub struct HealthService { - inner: S, -} - -impl Service> for HealthService -where - S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, - S::Response: 'static, - S::Error: Into + 'static, - S::Future: Send + 'static, -{ - type Response = HttpResponse; - type Error = BoxError; - type Future = Either< - Pin> + Send + 'static>>, - S::Future, - >; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.poll_ready(cx).map_err(Into::into) - } - - fn call(&mut self, request: HttpRequest) -> Self::Future { - if request.uri().path() == "/healthz" { - Either::A(Self::healthz().boxed()) - } else { - Either::B(self.inner.call(request)) - } - } -} - -impl HealthService -where - S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, - S::Response: 'static, - S::Error: Into + 'static, - S::Future: Send + 'static, -{ - async fn healthz() -> Result { - Ok(HttpResponse::new(HttpBody::from("OK"))) - } -} diff --git a/src/integration/mod.rs b/src/integration/mod.rs index c326c9c2..54fb6e4c 100644 --- a/src/integration/mod.rs +++ b/src/integration/mod.rs @@ -1,4 +1,5 @@ -use crate::client::auth::{AuthClientLayer, AuthClientService}; +use crate::client::auth::AuthLayer; +use crate::client::rpc::RpcClientService; use crate::debug_api::DebugClient; use crate::server::EngineApiClient; use crate::server::PayloadSource; @@ -9,7 +10,6 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadAttributes, PayloadId, PayloadStatus, PayloadStatusEnum, }; -use jsonrpsee::http_client::{HttpClient, transport::HttpBackend}; use jsonrpsee::proc_macros::rpc; use lazy_static::lazy_static; use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelopeV3, OpPayloadAttributes}; @@ -466,13 +466,13 @@ impl Drop for IntegrationFramework { } pub struct EngineApi { - pub engine_api_client: HttpClient>, + pub engine_api_client: RpcClientService, } impl EngineApi { pub fn new(url: &str, secret: &str) -> Result> { - let secret_layer = AuthClientLayer::new(JwtSecret::from_str(secret)?); - let middleware = tower::ServiceBuilder::default().layer(secret_layer); + let auth_layer = AuthLayer::new(JwtSecret::from_str(secret)?); + let middleware = tower::ServiceBuilder::default().layer(auth_layer); let client = jsonrpsee::http_client::HttpClientBuilder::default() .set_http_middleware(middleware) .build(url) diff --git a/src/main.rs b/src/main.rs index 17c13c36..b78c52db 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,8 +3,8 @@ use crate::client::rpc::{BuilderArgs, L2ClientArgs, RpcClient}; use ::tracing::{Level, info}; use clap::{Parser, Subcommand, arg}; use debug_api::DebugClient; -use health::HealthLayer; use metrics::init_metrics; +use probe::ProbeLayer; use std::net::SocketAddr; use tracing::init_tracing; @@ -20,10 +20,10 @@ use tokio::signal::unix::{SignalKind, signal as unix_signal}; mod client; mod debug_api; -mod health; #[cfg(all(feature = "integration", test))] mod integration; mod metrics; +mod probe; mod proxy; mod server; mod tracing; @@ -204,11 +204,18 @@ async fn main() -> eyre::Result<()> { info!("Boost sync enabled"); } + let (probe_layer, probes) = ProbeLayer::new(); + + probes.set_health(true); + probes.set_ready(true); + probes.set_live(true); + let rollup_boost = RollupBoostServer::new( l2_client, builder_client, boost_sync_enabled, args.execution_mode, + probes, ); // Spawn the debug server @@ -219,12 +226,14 @@ async fn main() -> eyre::Result<()> { // Build and start the server info!("Starting server on :{}", args.rpc_port); - let http_middleware = tower::ServiceBuilder::new().layer(ProxyLayer::new( - l2_client_args.l2_url, - l2_auth_jwt, - builder_args.builder_url, - builder_auth_jwt, - )); + let http_middleware = tower::ServiceBuilder::new() + .layer(probe_layer) + .layer(ProxyLayer::new( + l2_client_args.l2_url, + l2_auth_jwt, + builder_args.builder_url, + builder_auth_jwt, + )); let server = Server::builder() .set_http_middleware(http_middleware) diff --git a/src/probe.rs b/src/probe.rs new file mode 100644 index 00000000..e7b50bd5 --- /dev/null +++ b/src/probe.rs @@ -0,0 +1,155 @@ +use std::{ + pin::Pin, + sync::{Arc, atomic::AtomicBool}, + task::{Context, Poll}, +}; + +use futures::FutureExt as _; +use jsonrpsee::{ + core::BoxError, + http_client::{HttpBody, HttpRequest, HttpResponse}, +}; +use tower::{Layer, Service}; + +#[derive(Debug, Default)] +pub struct Probes { + pub health: AtomicBool, + pub ready: AtomicBool, + pub live: AtomicBool, +} + +impl Probes { + pub fn set_health(&self, value: bool) { + self.health + .store(value, std::sync::atomic::Ordering::Relaxed); + } + + pub fn set_ready(&self, value: bool) { + self.ready + .store(value, std::sync::atomic::Ordering::Relaxed); + } + + pub fn set_live(&self, value: bool) { + self.live.store(value, std::sync::atomic::Ordering::Relaxed); + } + + pub fn health(&self) -> bool { + self.health.load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn ready(&self) -> bool { + self.ready.load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn live(&self) -> bool { + self.live.load(std::sync::atomic::Ordering::Relaxed) + } +} + +/// A [`Layer`] that filters out /healthz requests and responds with a 200 OK. +#[derive(Clone, Debug)] +pub struct ProbeLayer { + probes: Arc, +} + +impl ProbeLayer { + pub(crate) fn new() -> (Self, Arc) { + let probes = Arc::new(Probes::default()); + ( + Self { + probes: probes.clone(), + }, + probes, + ) + } +} + +impl Layer for ProbeLayer { + type Service = ProbeService; + + fn layer(&self, inner: S) -> Self::Service { + ProbeService { + inner, + probes: self.probes.clone(), + } + } +} + +#[derive(Clone, Debug)] +pub struct ProbeService { + inner: S, + probes: Arc, +} + +impl Service> for ProbeService +where + S: Service, Response = HttpResponse> + Send + Sync + Clone + 'static, + S::Response: 'static, + S::Error: Into + 'static, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = BoxError; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, request: HttpRequest) -> Self::Future { + // See https://github.com/tower-rs/tower/blob/abb375d08cf0ba34c1fe76f66f1aba3dc4341013/tower-service/src/lib.rs#L276 + // for an explanation of this pattern + let mut service = self.clone(); + service.inner = std::mem::replace(&mut self.inner, service.inner); + + async move { + match request.uri().path() { + "/healthz" => { + if service.probes.health() { + ok() + } else { + internal_server_error() + } + } + "/readyz" => { + if service.probes.ready() { + ok() + } else { + service_unavailable() + } + } + "/livez" => { + if service.probes.live() { + ok() + } else { + service_unavailable() + } + } + _ => service.inner.call(request).await.map_err(|e| e.into()), + } + } + .boxed() + } +} + +fn ok() -> Result, BoxError> { + Ok(HttpResponse::builder() + .status(200) + .body(HttpBody::from("OK")) + .unwrap()) +} + +fn internal_server_error() -> Result, BoxError> { + Ok(HttpResponse::builder() + .status(500) + .body(HttpBody::from("Internal Server Error")) + .unwrap()) +} + +fn service_unavailable() -> Result, BoxError> { + Ok(HttpResponse::builder() + .status(503) + .body(HttpBody::from("Service Unavailable")) + .unwrap()) +} diff --git a/src/proxy.rs b/src/proxy.rs index 977bab4e..6e778f74 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -1,15 +1,13 @@ -use crate::HealthLayer; use crate::client::http::HttpClient; -use crate::health::HealthService; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; +use futures::FutureExt as _; use http::Uri; use jsonrpsee::core::{BoxError, http_helpers}; use jsonrpsee::http_client::{HttpBody, HttpRequest, HttpResponse}; use std::task::{Context, Poll}; use std::{future::Future, pin::Pin}; use tower::{Layer, Service}; -use tracing::info; const MULTIPLEX_METHODS: [&str; 4] = [ "engine_", @@ -17,6 +15,7 @@ const MULTIPLEX_METHODS: [&str; 4] = [ "eth_sendRawTransaction", "miner_", ]; + const FORWARD_REQUESTS: [&str; 6] = [ "eth_sendRawTransaction", "eth_sendRawTransactionConditional", @@ -51,7 +50,7 @@ impl ProxyLayer { } impl Layer for ProxyLayer { - type Service = HealthService>; + type Service = ProxyService; fn layer(&self, inner: S) -> Self::Service { let l2_client = HttpClient::new( @@ -66,13 +65,11 @@ impl Layer for ProxyLayer { PayloadSource::Builder, ); - let proxy = ProxyService { + ProxyService { inner, l2_client, builder_client, - }; - - HealthLayer.layer(proxy) + } } } @@ -112,44 +109,57 @@ where let mut service = self.clone(); service.inner = std::mem::replace(&mut self.inner, service.inner); - let fut = async move { - let (parts, body) = req.into_parts(); - let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; - - // Deserialize the bytes to find the method - let method = serde_json::from_slice::(&body_bytes)? - .method - .to_string(); - - if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { - if FORWARD_REQUESTS.contains(&method.as_str()) { - let builder_req = - HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); - let builder_method = method.clone(); - let mut builder_client = service.builder_client.clone(); - tokio::spawn(async move { - let _ = builder_client.forward(builder_req, builder_method).await; - }); - - let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - service.l2_client.forward(l2_req, method).await + async move { + let res = async move { + let (parts, body) = req.into_parts(); + let (body_bytes, _) = + http_helpers::read_body(&parts.headers, body, u32::MAX).await?; + + // Deserialize the bytes to find the method + let method = serde_json::from_slice::(&body_bytes)? + .method + .to_string(); + + if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { + if FORWARD_REQUESTS.contains(&method.as_str()) { + let builder_req = HttpRequest::from_parts( + parts.clone(), + HttpBody::from(body_bytes.clone()), + ); + let builder_method = method.clone(); + let mut builder_client = service.builder_client.clone(); + tokio::spawn(async move { + let _ = builder_client.forward(builder_req, builder_method).await; + }); + + let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + service.l2_client.forward(l2_req, method).await + } else { + let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + service.inner.call(req).await.map_err(|e| e.into()) + } } else { let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - info!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); - service.inner.call(req).await.map_err(|e| e.into()) + service.l2_client.forward(req, method).await } - } else { - let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - service.l2_client.forward(req, method).await } - }; - Box::pin(fut) + .await; + + Ok(res.unwrap_or_else(|e| { + HttpResponse::builder() + .status(500) + .body(HttpBody::from(e.to_string())) + .unwrap() + })) + } + .boxed() } } #[cfg(test)] mod tests { + use crate::probe::ProbeLayer; + use super::*; use alloy_primitives::{B256, Bytes, U64, U128, hex}; use alloy_rpc_types_engine::JwtSecret; @@ -166,7 +176,6 @@ mod tests { http_client::HttpClient, rpc_params, server::{ServerBuilder, ServerHandle}, - types::{ErrorCode, ErrorObject}, }; use serde_json::json; use std::{ @@ -377,11 +386,6 @@ mod tests { async fn proxy_failure() { let response = send_request("non_existent_method").await; assert!(response.is_err()); - let expected_error = ErrorObject::from(ErrorCode::MethodNotFound).into_owned(); - assert!(matches!( - response.unwrap_err(), - ClientError::Call(e) if e == expected_error - )); } async fn does_not_proxy_engine_method() { @@ -465,11 +469,20 @@ mod tests { .parse::() .unwrap(); + let (probe_layer, probes) = ProbeLayer::new(); + probes.set_health(true); + probes.set_ready(true); + probes.set_live(true); + let proxy_layer = ProxyLayer::new(l2_auth_uri.clone(), jwt, l2_auth_uri, jwt); // Create a layered server let server = ServerBuilder::default() - .set_http_middleware(tower::ServiceBuilder::new().layer(proxy_layer)) + .set_http_middleware( + tower::ServiceBuilder::new() + .layer(probe_layer) + .layer(proxy_layer), + ) .build(addr.parse::().unwrap()) .await .unwrap(); diff --git a/src/server.rs b/src/server.rs index 36bb3381..0428e07c 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,5 +1,6 @@ use crate::client::rpc::RpcClient; use crate::debug_api::DebugServer; +use crate::probe::Probes; use alloy_primitives::B256; use moka::sync::Cache; use opentelemetry::trace::SpanKind; @@ -119,6 +120,7 @@ pub struct RollupBoostServer { pub boost_sync: bool, pub payload_trace_context: Arc, execution_mode: Arc>, + probes: Arc, } impl RollupBoostServer { @@ -127,6 +129,7 @@ impl RollupBoostServer { builder_client: RpcClient, boost_sync: bool, initial_execution_mode: ExecutionMode, + probes: Arc, ) -> Self { Self { l2_client: Arc::new(l2_client), @@ -134,6 +137,7 @@ impl RollupBoostServer { boost_sync, payload_trace_context: Arc::new(PayloadTraceContext::new()), execution_mode: Arc::new(Mutex::new(initial_execution_mode)), + probes, } } @@ -342,7 +346,12 @@ impl EngineApiServer for RollupBoostServer { let (l2_payload, builder_payload) = tokio::join!(l2_client_future, builder_client_future); let (payload, context) = match (builder_payload, l2_payload) { - (Ok(Some(builder)), _) => Ok((builder, "builder")), + (Ok(Some(builder)), _) => { + self.probes.set_live(true); + self.probes.set_ready(true); + self.probes.set_health(true); + Ok((builder, "builder")) + } (_, Ok(l2)) => Ok((l2, "l2")), (_, Err(e)) => Err(e), }?; @@ -420,6 +429,8 @@ impl EngineApiServer for RollupBoostServer { #[cfg(test)] #[allow(clippy::complexity)] mod tests { + use crate::probe::ProbeLayer; + use super::*; use alloy_primitives::hex; use alloy_primitives::{FixedBytes, U256}; @@ -529,11 +540,14 @@ mod tests { let builder_client = RpcClient::new(builder_auth_rpc, jwt_secret, 2000, PayloadSource::Builder).unwrap(); + let (_probe_layer, probes) = ProbeLayer::new(); + let rollup_boost_client = RollupBoostServer::new( l2_client, builder_client, boost_sync, ExecutionMode::Enabled, + probes, ); let module: RpcModule<()> = rollup_boost_client.try_into().unwrap(); From 121bb2b6b66f5f29a9f4eeb5c189842b8b05163e Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 19:21:18 -0700 Subject: [PATCH 11/39] implement health/ready check logic --- src/client/rpc.rs | 9 +++++++++ src/main.rs | 16 ++++++++++------ src/probe.rs | 16 +++++++++++----- src/proxy.rs | 1 - src/server.rs | 39 +++++++++++++++++++++++++++++++++++---- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/client/rpc.rs b/src/client/rpc.rs index e12c4423..79c0c69e 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -7,12 +7,15 @@ use alloy_rpc_types_engine::{ }; use clap::{Parser, arg}; use http::Uri; +use jsonrpsee::core::client::ClientT; use jsonrpsee::http_client::transport::HttpBackend; use jsonrpsee::http_client::{HttpClient, HttpClientBuilder}; +use jsonrpsee::rpc_params; use jsonrpsee::types::ErrorObjectOwned; use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelopeV3, OpPayloadAttributes}; use opentelemetry::trace::SpanKind; use paste::paste; +use serde_json::Value; use std::path::PathBuf; use std::time::Duration; use thiserror::Error; @@ -142,6 +145,12 @@ impl RpcClient { Ok(res) } + + pub async fn health(&self) -> ClientResult<()> { + let method = "web3_clientVersion"; + let _res: Value = self.auth_client.request(method, rpc_params![]).await?; + Ok(()) + } } /// Generates Clap argument structs with a prefix to create a unique namespace when specifing RPC client config via the CLI. diff --git a/src/main.rs b/src/main.rs index b78c52db..1973df6c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -206,16 +206,12 @@ async fn main() -> eyre::Result<()> { let (probe_layer, probes) = ProbeLayer::new(); - probes.set_health(true); - probes.set_ready(true); - probes.set_live(true); - let rollup_boost = RollupBoostServer::new( - l2_client, + l2_client.clone(), builder_client, boost_sync_enabled, args.execution_mode, - probes, + probes.clone(), ); // Spawn the debug server @@ -241,6 +237,14 @@ async fn main() -> eyre::Result<()> { .await?; let handle = server.start(module); + while l2_client.health().await.is_err() { + info!("waiting for l2 client to be healthy"); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + + probes.set_ready(true); + probes.set_health(true); + let stop_handle = handle.clone(); // Capture SIGINT and SIGTERM diff --git a/src/probe.rs b/src/probe.rs index e7b50bd5..fedef31a 100644 --- a/src/probe.rs +++ b/src/probe.rs @@ -11,13 +11,23 @@ use jsonrpsee::{ }; use tower::{Layer, Service}; -#[derive(Debug, Default)] +#[derive(Debug)] pub struct Probes { pub health: AtomicBool, pub ready: AtomicBool, pub live: AtomicBool, } +impl Default for Probes { + fn default() -> Self { + Self { + health: AtomicBool::new(false), + ready: AtomicBool::new(false), + live: AtomicBool::new(true), + } + } +} + impl Probes { pub fn set_health(&self, value: bool) { self.health @@ -29,10 +39,6 @@ impl Probes { .store(value, std::sync::atomic::Ordering::Relaxed); } - pub fn set_live(&self, value: bool) { - self.live.store(value, std::sync::atomic::Ordering::Relaxed); - } - pub fn health(&self) -> bool { self.health.load(std::sync::atomic::Ordering::Relaxed) } diff --git a/src/proxy.rs b/src/proxy.rs index 6e778f74..02f5c41b 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -472,7 +472,6 @@ mod tests { let (probe_layer, probes) = ProbeLayer::new(); probes.set_health(true); probes.set_ready(true); - probes.set_live(true); let proxy_layer = ProxyLayer::new(l2_auth_uri.clone(), jwt, l2_auth_uri, jwt); diff --git a/src/server.rs b/src/server.rs index 0428e07c..b44a8064 100644 --- a/src/server.rs +++ b/src/server.rs @@ -6,6 +6,7 @@ use moka::sync::Cache; use opentelemetry::trace::SpanKind; use parking_lot::Mutex; use std::sync::Arc; +use std::sync::atomic::AtomicU32; use alloy_rpc_types_engine::{ ExecutionPayload, ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadId, @@ -23,6 +24,7 @@ use tracing::{debug, info, instrument}; use jsonrpsee::proc_macros::rpc; const CACHE_SIZE: u64 = 100; +const BUILDER_MISSED_PAYLOAD_THRESHOLD: u32 = 3; pub struct PayloadTraceContext { block_hash_to_payload_ids: Cache>, @@ -121,6 +123,8 @@ pub struct RollupBoostServer { pub payload_trace_context: Arc, execution_mode: Arc>, probes: Arc, + /// Number of consecutive builder failures + builder_failures: Arc, } impl RollupBoostServer { @@ -138,9 +142,21 @@ impl RollupBoostServer { payload_trace_context: Arc::new(PayloadTraceContext::new()), execution_mode: Arc::new(Mutex::new(initial_execution_mode)), probes, + builder_failures: Default::default(), } } + pub fn increment_builder_failures(&self) -> u32 { + self.builder_failures + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + + 1 + } + + pub fn reset_builder_failures(&self) { + self.builder_failures + .store(0, std::sync::atomic::Ordering::Relaxed); + } + pub async fn start_debug_server(&self, debug_addr: &str) -> eyre::Result<()> { let server = DebugServer::new(self.execution_mode.clone()); server.run(debug_addr).await?; @@ -347,13 +363,28 @@ impl EngineApiServer for RollupBoostServer { let (l2_payload, builder_payload) = tokio::join!(l2_client_future, builder_client_future); let (payload, context) = match (builder_payload, l2_payload) { (Ok(Some(builder)), _) => { - self.probes.set_live(true); - self.probes.set_ready(true); + // builder successfully returned a payload + // set health and reset any past failures self.probes.set_health(true); + self.probes.set_ready(true); + self.reset_builder_failures(); Ok((builder, "builder")) } - (_, Ok(l2)) => Ok((l2, "l2")), - (_, Err(e)) => Err(e), + (_, Ok(l2)) => { + // builder failed to return a payload + if self.increment_builder_failures() > BUILDER_MISSED_PAYLOAD_THRESHOLD { + self.probes.set_health(false); + self.probes.set_ready(true); + } + Ok((l2, "l2")) + } + (_, Err(e)) => { + // builder and l2 failed to return a payload + // set both health and ready to false + self.probes.set_health(false); + self.probes.set_ready(false); + Err(e) + } }?; let inner_payload = ExecutionPayload::from(payload.clone().execution_payload); From 2e1ce26f9a438e4ccda6a43f9c9cd03b9cd9143d Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 19:25:15 -0700 Subject: [PATCH 12/39] modify ready logic --- src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index b44a8064..b5c9940f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -372,9 +372,9 @@ impl EngineApiServer for RollupBoostServer { } (_, Ok(l2)) => { // builder failed to return a payload + self.probes.set_ready(true); if self.increment_builder_failures() > BUILDER_MISSED_PAYLOAD_THRESHOLD { self.probes.set_health(false); - self.probes.set_ready(true); } Ok((l2, "l2")) } From 04678f79c9724c85dd6528f0fbe8ca6cd7ab2ae1 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 18 Mar 2025 19:40:21 -0700 Subject: [PATCH 13/39] fix comment/feature --- Cargo.toml | 1 - src/probe.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 40143552..3b3ac90b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,5 +69,4 @@ reth-rpc-layer = { git = "https://github.com/paradigmxyz/reth.git", rev = "v1.3. ctor = "0.4.1" [features] -default = ["integration"] integration = [] diff --git a/src/probe.rs b/src/probe.rs index fedef31a..96e8f03e 100644 --- a/src/probe.rs +++ b/src/probe.rs @@ -52,7 +52,7 @@ impl Probes { } } -/// A [`Layer`] that filters out /healthz requests and responds with a 200 OK. +/// A [`Layer`] that adds probe endpoints to a service. #[derive(Clone, Debug)] pub struct ProbeLayer { probes: Arc, From 7a3e0a096f10db04e6822d9f676f77902ab651a0 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 19 Mar 2025 09:50:53 -0700 Subject: [PATCH 14/39] delete old file --- src/auth_layer.rs | 81 ----------------------------------------------- 1 file changed, 81 deletions(-) delete mode 100644 src/auth_layer.rs diff --git a/src/auth_layer.rs b/src/auth_layer.rs deleted file mode 100644 index af39571c..00000000 --- a/src/auth_layer.rs +++ /dev/null @@ -1,81 +0,0 @@ -// From reth_rpc_layer -use alloy_rpc_types_engine::{Claims, JwtSecret}; -use http::{HeaderValue, header::AUTHORIZATION}; -use std::{ - task::{Context, Poll}, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; -use tower::{Layer, Service}; - -/// A layer that adds a new JWT token to every request using `AuthClientService`. -#[derive(Debug)] -pub struct AuthClientLayer { - secret: JwtSecret, -} - -impl AuthClientLayer { - /// Create a new `AuthClientLayer` with the given `secret`. - pub const fn new(secret: JwtSecret) -> Self { - Self { secret } - } -} - -impl Layer for AuthClientLayer { - type Service = AuthClientService; - - fn layer(&self, inner: S) -> Self::Service { - AuthClientService::new(self.secret, inner) - } -} - -/// Automatically authenticates every client request with the given `secret`. -#[derive(Debug, Clone)] -pub struct AuthClientService { - secret: JwtSecret, - inner: S, -} - -impl AuthClientService { - const fn new(secret: JwtSecret, inner: S) -> Self { - Self { secret, inner } - } -} - -impl Service> for AuthClientService -where - S: Service>, -{ - type Response = S::Response; - type Error = S::Error; - type Future = S::Future; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.poll_ready(cx) - } - - fn call(&mut self, mut request: http::Request) -> Self::Future { - request - .headers_mut() - .insert(AUTHORIZATION, secret_to_bearer_header(&self.secret)); - self.inner.call(request) - } -} - -/// Helper function to convert a secret into a Bearer auth header value with claims according to -/// . -/// The token is valid for 60 seconds. -pub fn secret_to_bearer_header(secret: &JwtSecret) -> HeaderValue { - format!( - "Bearer {}", - secret - .encode(&Claims { - iat: (SystemTime::now().duration_since(UNIX_EPOCH).unwrap() - + Duration::from_secs(60)) - .as_secs(), - exp: None, - }) - .unwrap() - ) - .parse() - .unwrap() -} From 0a1d9ebd6f16db7d31fcf3a5e5fc1a333848ac65 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Thu, 20 Mar 2025 23:24:53 -0700 Subject: [PATCH 15/39] working --- Cargo.lock | 23 ++++++++++- Cargo.toml | 1 + Dockerfile | 4 +- src/auth_layer.rs | 81 ------------------------------------ src/client/http.rs | 101 ++++++++++++++------------------------------- 5 files changed, 56 insertions(+), 154 deletions(-) delete mode 100644 src/auth_layer.rs diff --git a/Cargo.lock b/Cargo.lock index 8c5b0592..bb9fe5cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3755,7 +3755,7 @@ dependencies = [ "jsonrpsee-http-client", "pin-project", "tower 0.4.13", - "tower-http", + "tower-http 0.6.2", "tracing", ] @@ -3855,6 +3855,7 @@ dependencies = [ "tokio", "tokio-util", "tower 0.4.13", + "tower-http 0.5.2", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -4856,6 +4857,26 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "async-compression", + "bitflags 2.9.0", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.6.2" diff --git a/Cargo.toml b/Cargo.toml index 85c66977..e40afd23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ reqwest = "0.12.5" http = "1.1.0" dotenv = "0.15.0" tower = "0.4.13" +tower-http = { version = "0.5.2", features = ["decompression-full"] } http-body = "1.0.1" http-body-util = "0.1.2" hyper = { version = "1.4.1", features = ["full"] } diff --git a/Dockerfile b/Dockerfile index cf63059a..d1354873 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ # # Based on https://depot.dev/blog/rust-dockerfile-best-practices # -FROM rust:1.82 AS base +FROM rust:1.85.1 AS base ARG FEATURES @@ -61,4 +61,4 @@ WORKDIR /app ARG ROLLUP_BOOST_BIN="rollup-boost" COPY --from=builder /app/target/release/${ROLLUP_BOOST_BIN} /usr/local/bin/ -ENTRYPOINT ["/usr/local/bin/rollup-boost"] \ No newline at end of file +ENTRYPOINT ["/usr/local/bin/rollup-boost"] diff --git a/src/auth_layer.rs b/src/auth_layer.rs deleted file mode 100644 index af39571c..00000000 --- a/src/auth_layer.rs +++ /dev/null @@ -1,81 +0,0 @@ -// From reth_rpc_layer -use alloy_rpc_types_engine::{Claims, JwtSecret}; -use http::{HeaderValue, header::AUTHORIZATION}; -use std::{ - task::{Context, Poll}, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; -use tower::{Layer, Service}; - -/// A layer that adds a new JWT token to every request using `AuthClientService`. -#[derive(Debug)] -pub struct AuthClientLayer { - secret: JwtSecret, -} - -impl AuthClientLayer { - /// Create a new `AuthClientLayer` with the given `secret`. - pub const fn new(secret: JwtSecret) -> Self { - Self { secret } - } -} - -impl Layer for AuthClientLayer { - type Service = AuthClientService; - - fn layer(&self, inner: S) -> Self::Service { - AuthClientService::new(self.secret, inner) - } -} - -/// Automatically authenticates every client request with the given `secret`. -#[derive(Debug, Clone)] -pub struct AuthClientService { - secret: JwtSecret, - inner: S, -} - -impl AuthClientService { - const fn new(secret: JwtSecret, inner: S) -> Self { - Self { secret, inner } - } -} - -impl Service> for AuthClientService -where - S: Service>, -{ - type Response = S::Response; - type Error = S::Error; - type Future = S::Future; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.poll_ready(cx) - } - - fn call(&mut self, mut request: http::Request) -> Self::Future { - request - .headers_mut() - .insert(AUTHORIZATION, secret_to_bearer_header(&self.secret)); - self.inner.call(request) - } -} - -/// Helper function to convert a secret into a Bearer auth header value with claims according to -/// . -/// The token is valid for 60 seconds. -pub fn secret_to_bearer_header(secret: &JwtSecret) -> HeaderValue { - format!( - "Bearer {}", - secret - .encode(&Claims { - iat: (SystemTime::now().duration_since(UNIX_EPOCH).unwrap() - + Duration::from_secs(60)) - .as_secs(), - exp: None, - }) - .unwrap() - ) - .parse() - .unwrap() -} diff --git a/src/client/http.rs b/src/client/http.rs index 5de2a3c1..c41ff63f 100644 --- a/src/client/http.rs +++ b/src/client/http.rs @@ -1,10 +1,7 @@ -use crate::client::auth::{AuthClientLayer, AuthClientService}; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; -use eyre::Context; -use flate2::read::GzDecoder; -use http::response::Parts; -use http::{Request, Uri}; +use eyre::bail; +use http::Uri; use http_body_util::BodyExt; use hyper_rustls::HttpsConnector; use hyper_util::client::legacy::Client; @@ -13,13 +10,15 @@ use hyper_util::rt::TokioExecutor; use jsonrpsee::core::BoxError; use jsonrpsee::http_client::HttpBody; use opentelemetry::trace::SpanKind; -use std::io::Read; -use tower::{Layer, Service}; +use tower::{Service as _, ServiceBuilder, ServiceExt}; +use tower_http::decompression::{Decompression, DecompressionLayer}; use tracing::{debug, error, instrument}; +use super::auth::{AuthClientLayer, AuthClientService}; + #[derive(Clone, Debug)] pub(crate) struct HttpClient { - client: AuthClientService, HttpBody>>, + client: Decompression, HttpBody>>>, url: Uri, target: PayloadSource, } @@ -33,10 +32,14 @@ impl HttpClient { .enable_http1() .enable_http2() .build(); - let auth = AuthClientLayer::new(secret); - let client: Client, HttpBody> = - Client::builder(TokioExecutor::new()).build(connector); - let client = auth.layer(client); + + let client = Client::builder(TokioExecutor::new()).build(connector); + + let client = ServiceBuilder::new() + .layer(DecompressionLayer::new()) + .layer(AuthClientLayer::new(secret)) + .service(client); + Self { client, url, @@ -44,79 +47,40 @@ impl HttpClient { } } + /// Forwards an HTTP request to the `authrpc``, attaching the provided JWT authorization. #[instrument( skip(self, req), fields(otel.kind = ?SpanKind::Client), - err + err(Debug) )] - pub(crate) async fn forward( + pub async fn forward( &mut self, - mut req: Request, + mut req: http::Request, method: String, ) -> Result, BoxError> { - *req.uri_mut() = self.url.clone(); debug!("forwarding {} to {}", method, self.target); + *req.uri_mut() = self.url.clone(); - let res = self.client.call(req).await?; + let res = self.client.ready().await?.call(req).await?; let (parts, body) = res.into_parts(); - let body_bytes = body - .collect() - .await - .map_err(|e| { - error!( - target: "proxy::forward_request", - message = "error collecting body", - error = %e, - ); - e - })? - .to_bytes() - .to_vec(); - let parts_clone = parts.clone(); - let body_bytes_clone = body_bytes.clone(); + let body_bytes = body.collect().await?.to_bytes().to_vec(); - self.handle_response(parts_clone, body_bytes_clone) - .context("error fowarding request")?; + if let Err(e) = parse_response_code(&body_bytes) { + error!( + error = %e, + "error in forwarded response", + ); + } Ok(http::Response::from_parts( parts, HttpBody::from(body_bytes), )) } - - fn handle_response(&self, parts: Parts, body_bytes: Vec) -> eyre::Result<()> { - // Check for GZIP compression - let is_gzipped = parts - .headers - .get(http::header::CONTENT_ENCODING) - .is_some_and(|val| val.as_bytes() == b"gzip"); - - let decoded_body = if is_gzipped { - // Decompress GZIP content - let mut decoder = GzDecoder::new(&body_bytes[..]); - let mut decoded = Vec::new(); - decoder - .read_to_end(&mut decoded) - .context("error decompressing body")?; - decoded - } else { - body_bytes - }; - - debug!( - target: "proxy::forward_request", - message = "raw response body", - body = %String::from_utf8_lossy(&decoded_body), - ); - - handle_response_code(&decoded_body)?; - - Ok(()) - } } -fn handle_response_code(body_bytes: &[u8]) -> eyre::Result<()> { +fn parse_response_code(body_bytes: &[u8]) -> eyre::Result<()> { #[derive(serde::Deserialize, Debug)] struct RpcResponse { error: Option, @@ -127,12 +91,9 @@ fn handle_response_code(body_bytes: &[u8]) -> eyre::Result<()> { code: i32, } - let res = - serde_json::from_slice::(body_bytes).context("error deserializing body")?; - + let res = serde_json::from_slice::(body_bytes)?; if let Some(e) = res.error { - eyre::bail!("code: {}", e.code) + bail!("code: {}", e.code); } - Ok(()) } From adbf5f36ac74a9065c22994ceac3bde603b71c9b Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Mon, 24 Mar 2025 23:26:28 -0700 Subject: [PATCH 16/39] Update src/client/http.rs Co-authored-by: shana --- src/client/http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/http.rs b/src/client/http.rs index c41ff63f..e9cf94f3 100644 --- a/src/client/http.rs +++ b/src/client/http.rs @@ -47,7 +47,7 @@ impl HttpClient { } } - /// Forwards an HTTP request to the `authrpc``, attaching the provided JWT authorization. + /// Forwards an HTTP request to the `authrpc`, attaching the provided JWT authorization. #[instrument( skip(self, req), fields(otel.kind = ?SpanKind::Client), From f5e8f69f5d926c262b30d8992efcbfad5f64cfd7 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 25 Mar 2025 08:29:30 -0700 Subject: [PATCH 17/39] parse response cod --- src/client/http.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/client/http.rs b/src/client/http.rs index e9cf94f3..9ae3e26d 100644 --- a/src/client/http.rs +++ b/src/client/http.rs @@ -66,11 +66,8 @@ impl HttpClient { let (parts, body) = res.into_parts(); let body_bytes = body.collect().await?.to_bytes().to_vec(); - if let Err(e) = parse_response_code(&body_bytes) { - error!( - error = %e, - "error in forwarded response", - ); + if let Some(code) = parse_response_code(&body_bytes)? { + error!(%code, "error in forwarded response"); } Ok(http::Response::from_parts( @@ -80,7 +77,7 @@ impl HttpClient { } } -fn parse_response_code(body_bytes: &[u8]) -> eyre::Result<()> { +fn parse_response_code(body_bytes: &[u8]) -> eyre::Result> { #[derive(serde::Deserialize, Debug)] struct RpcResponse { error: Option, @@ -92,8 +89,6 @@ fn parse_response_code(body_bytes: &[u8]) -> eyre::Result<()> { } let res = serde_json::from_slice::(body_bytes)?; - if let Some(e) = res.error { - bail!("code: {}", e.code); - } - Ok(()) + + Ok(res.error.map(|e| e.code)) } From 7497301f99c57665570aefe1ad6e4b7caf15a4bf Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 25 Mar 2025 08:34:57 -0700 Subject: [PATCH 18/39] clippy fix --- src/client/http.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/http.rs b/src/client/http.rs index 9ae3e26d..892def05 100644 --- a/src/client/http.rs +++ b/src/client/http.rs @@ -1,6 +1,5 @@ use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; -use eyre::bail; use http::Uri; use http_body_util::BodyExt; use hyper_rustls::HttpsConnector; From ddd6d9ba6d9ecd50d33c4da4af2f11dfca4ac74c Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 28 Mar 2025 14:24:21 -0700 Subject: [PATCH 19/39] Probe docs --- docs/running-rollup-boost.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/running-rollup-boost.md b/docs/running-rollup-boost.md index 8991f913..14543058 100644 --- a/docs/running-rollup-boost.md +++ b/docs/running-rollup-boost.md @@ -28,6 +28,16 @@ While this does not ensure high availability for the builder, the chain will hav ![rollup-boost-op-conductor](../assets/rollup-boost-op-conductor.png) +### Health Checks + +`rollup-boost` supports the standard array of kubernetes probes: + +- `/healthz` determines wether everything is 100% up and running. If the builder fails to produce paylaods the healthz endpoint will return an error. `op-conductor` should eventually be able to use this signal to switch to a different sequencer in an HA sequencer setup. In a future upgrade to `op-conductor`, A sequencer leader with a healthy EL (`rollup-boost` in our case) could be selected preferentially over one with an unhealthy EL. + +- `/readyz` returns true as long as we're effectively building payloads from the l2 client. This means that we still produce blocks with this instance of rollup-boost. In an HA sequencer setup, if no ELs are healthy, then the ready probe can instead be used to select the sequencer leader. + +- `/livez` determines wether or not `rollup-boost` is live (running and not deadlocked) and responding to requests. If `rollup-boost` fails to respond, kubernetes can use this as a signal to restart the pod. + ## Observability To check if the rollup-boost server is running, you can check the health endpoint: From 9b6099112c8ce40672b6a3168f7b127288fe16ae Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 28 Mar 2025 15:18:19 -0700 Subject: [PATCH 20/39] Switch to returning health status only from /healthz using http status codes --- src/main.rs | 3 -- src/probe.rs | 108 +++++++++++++++++++++----------------------------- src/proxy.rs | 4 +- src/server.rs | 30 ++------------ 4 files changed, 51 insertions(+), 94 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1973df6c..cbfa25ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -242,9 +242,6 @@ async fn main() -> eyre::Result<()> { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } - probes.set_ready(true); - probes.set_health(true); - let stop_handle = handle.clone(); // Capture SIGINT and SIGTERM diff --git a/src/probe.rs b/src/probe.rs index 96e8f03e..645712f9 100644 --- a/src/probe.rs +++ b/src/probe.rs @@ -1,6 +1,6 @@ use std::{ pin::Pin, - sync::{Arc, atomic::AtomicBool}, + sync::Arc, task::{Context, Poll}, }; @@ -9,46 +9,44 @@ use jsonrpsee::{ core::BoxError, http_client::{HttpBody, HttpRequest, HttpResponse}, }; +use parking_lot::Mutex; use tower::{Layer, Service}; -#[derive(Debug)] -pub struct Probes { - pub health: AtomicBool, - pub ready: AtomicBool, - pub live: AtomicBool, +#[derive(Copy, Clone, Debug, Default)] +pub enum Health { + /// Indicates that the builder is building blocks + Healthy, + /// Indicates that the l2 is building blocks, but the builder is not + PartialContent, + /// Indicates that blocks are not being built by either the l2 or the builder + /// + /// Service starts out unavailable until the first blocks are built + #[default] + ServiceUnavailable, } -impl Default for Probes { - fn default() -> Self { - Self { - health: AtomicBool::new(false), - ready: AtomicBool::new(false), - live: AtomicBool::new(true), +impl From for HttpResponse { + fn from(health: Health) -> Self { + match health { + Health::Healthy => ok(), + Health::PartialContent => partial_content(), + Health::ServiceUnavailable => service_unavailable(), } } } -impl Probes { - pub fn set_health(&self, value: bool) { - self.health - .store(value, std::sync::atomic::Ordering::Relaxed); - } - - pub fn set_ready(&self, value: bool) { - self.ready - .store(value, std::sync::atomic::Ordering::Relaxed); - } - - pub fn health(&self) -> bool { - self.health.load(std::sync::atomic::Ordering::Relaxed) - } +#[derive(Debug, Default)] +pub struct Probes { + health: Mutex, +} - pub fn ready(&self) -> bool { - self.ready.load(std::sync::atomic::Ordering::Relaxed) +impl Probes { + pub fn set_health(&self, value: Health) { + *self.health.lock() = value; } - pub fn live(&self) -> bool { - self.live.load(std::sync::atomic::Ordering::Relaxed) + pub fn health(&self) -> Health { + *self.health.lock() } } @@ -111,27 +109,13 @@ where async move { match request.uri().path() { - "/healthz" => { - if service.probes.health() { - ok() - } else { - internal_server_error() - } - } - "/readyz" => { - if service.probes.ready() { - ok() - } else { - service_unavailable() - } - } - "/livez" => { - if service.probes.live() { - ok() - } else { - service_unavailable() - } - } + // Return health status + "/healthz" => Ok(service.probes.health().into()), + // Service is responding, and therefor ready + "/readyz" => Ok(ok()), + // Service is responding, and therefor live + "/livez" => Ok(ok()), + // Forward the request to the inner service _ => service.inner.call(request).await.map_err(|e| e.into()), } } @@ -139,23 +123,23 @@ where } } -fn ok() -> Result, BoxError> { - Ok(HttpResponse::builder() +fn ok() -> HttpResponse { + HttpResponse::builder() .status(200) .body(HttpBody::from("OK")) - .unwrap()) + .unwrap() } -fn internal_server_error() -> Result, BoxError> { - Ok(HttpResponse::builder() - .status(500) - .body(HttpBody::from("Internal Server Error")) - .unwrap()) +fn partial_content() -> HttpResponse { + HttpResponse::builder() + .status(206) + .body(HttpBody::from("Partial Content")) + .unwrap() } -fn service_unavailable() -> Result, BoxError> { - Ok(HttpResponse::builder() +fn service_unavailable() -> HttpResponse { + HttpResponse::builder() .status(503) .body(HttpBody::from("Service Unavailable")) - .unwrap()) + .unwrap() } diff --git a/src/proxy.rs b/src/proxy.rs index 02f5c41b..0a192c57 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -469,9 +469,7 @@ mod tests { .parse::() .unwrap(); - let (probe_layer, probes) = ProbeLayer::new(); - probes.set_health(true); - probes.set_ready(true); + let (probe_layer, _probes) = ProbeLayer::new(); let proxy_layer = ProxyLayer::new(l2_auth_uri.clone(), jwt, l2_auth_uri, jwt); diff --git a/src/server.rs b/src/server.rs index 966a1025..821c5f73 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,12 +1,11 @@ use crate::client::rpc::RpcClient; use crate::debug_api::DebugServer; -use crate::probe::Probes; +use crate::probe::{Health, Probes}; use alloy_primitives::B256; use moka::sync::Cache; use opentelemetry::trace::SpanKind; use parking_lot::Mutex; use std::sync::Arc; -use std::sync::atomic::AtomicU32; use alloy_rpc_types_engine::{ ExecutionPayload, ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadId, @@ -24,7 +23,6 @@ use tracing::{debug, info, instrument}; use jsonrpsee::proc_macros::rpc; const CACHE_SIZE: u64 = 100; -const BUILDER_MISSED_PAYLOAD_THRESHOLD: u32 = 3; pub struct PayloadTraceContext { block_hash_to_payload_ids: Cache>, @@ -123,8 +121,6 @@ pub struct RollupBoostServer { pub payload_trace_context: Arc, execution_mode: Arc>, probes: Arc, - /// Number of consecutive builder failures - builder_failures: Arc, } impl RollupBoostServer { @@ -142,21 +138,9 @@ impl RollupBoostServer { payload_trace_context: Arc::new(PayloadTraceContext::new()), execution_mode: Arc::new(Mutex::new(initial_execution_mode)), probes, - builder_failures: Default::default(), } } - pub fn increment_builder_failures(&self) -> u32 { - self.builder_failures - .fetch_add(1, std::sync::atomic::Ordering::Relaxed) - + 1 - } - - pub fn reset_builder_failures(&self) { - self.builder_failures - .store(0, std::sync::atomic::Ordering::Relaxed); - } - pub async fn start_debug_server(&self, debug_addr: &str) -> eyre::Result<()> { let server = DebugServer::new(self.execution_mode.clone()); server.run(debug_addr).await?; @@ -366,24 +350,18 @@ impl EngineApiServer for RollupBoostServer { (Ok(Some(builder)), _) => { // builder successfully returned a payload // set health and reset any past failures - self.probes.set_health(true); - self.probes.set_ready(true); - self.reset_builder_failures(); + self.probes.set_health(Health::Healthy); Ok((builder, PayloadSource::Builder)) } (_, Ok(l2)) => { // builder failed to return a payload - self.probes.set_ready(true); - if self.increment_builder_failures() > BUILDER_MISSED_PAYLOAD_THRESHOLD { - self.probes.set_health(false); - } + self.probes.set_health(Health::PartialContent); Ok((l2, PayloadSource::L2)) } (_, Err(e)) => { // builder and l2 failed to return a payload // set both health and ready to false - self.probes.set_health(false); - self.probes.set_ready(false); + self.probes.set_health(Health::ServiceUnavailable); Err(e) } }?; From 74d723ef9564e7c302c0ccc8f7c27d1c6ef3eb7c Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 28 Mar 2025 15:28:27 -0700 Subject: [PATCH 21/39] Update docs to describe health status codes --- docs/running-rollup-boost.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/running-rollup-boost.md b/docs/running-rollup-boost.md index 14543058..0f9f9dd3 100644 --- a/docs/running-rollup-boost.md +++ b/docs/running-rollup-boost.md @@ -32,11 +32,15 @@ While this does not ensure high availability for the builder, the chain will hav `rollup-boost` supports the standard array of kubernetes probes: -- `/healthz` determines wether everything is 100% up and running. If the builder fails to produce paylaods the healthz endpoint will return an error. `op-conductor` should eventually be able to use this signal to switch to a different sequencer in an HA sequencer setup. In a future upgrade to `op-conductor`, A sequencer leader with a healthy EL (`rollup-boost` in our case) could be selected preferentially over one with an unhealthy EL. +- `/healthz` Returns various status codes to communicate `rollup-boost` health + - 200 OK - The builder is producing blocks + - 206 Partial Content - The l2 is producing blocks, but the builder is not + - 503 Service Unavailable - Neither the l2 or the builder is producing blocks +`op-conductor` should eventually be able to use this signal to switch to a different sequencer in an HA sequencer setup. In a future upgrade to `op-conductor`, A sequencer leader with a healthy (200 OK) EL (`rollup-boost` in our case) could be selected preferentially over one with an unhealthy (206 or 503) EL. If no ELs are healthy, then we can fallback to an EL which is responding with `206 Partial Content`. -- `/readyz` returns true as long as we're effectively building payloads from the l2 client. This means that we still produce blocks with this instance of rollup-boost. In an HA sequencer setup, if no ELs are healthy, then the ready probe can instead be used to select the sequencer leader. +- `/readyz` Used by kubernetes to determine if the service is ready to accept traffic. Should always respond with `200 OK` -- `/livez` determines wether or not `rollup-boost` is live (running and not deadlocked) and responding to requests. If `rollup-boost` fails to respond, kubernetes can use this as a signal to restart the pod. +- `/livez` determines wether or not `rollup-boost` is live (running and not deadlocked) and responding to requests. If `rollup-boost` fails to respond, kubernetes can use this as a signal to restart the pod. Should always respond with `200 OK` ## Observability From 2732c3813b0bb1afd1781d75a079c70c5ec1c963 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 28 Mar 2025 15:32:30 -0700 Subject: [PATCH 22/39] remove stray comments --- src/server.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server.rs b/src/server.rs index 821c5f73..5b20034a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -349,7 +349,6 @@ impl EngineApiServer for RollupBoostServer { let (payload, context) = match (builder_payload, l2_payload) { (Ok(Some(builder)), _) => { // builder successfully returned a payload - // set health and reset any past failures self.probes.set_health(Health::Healthy); Ok((builder, PayloadSource::Builder)) } @@ -360,7 +359,6 @@ impl EngineApiServer for RollupBoostServer { } (_, Err(e)) => { // builder and l2 failed to return a payload - // set both health and ready to false self.probes.set_health(Health::ServiceUnavailable); Err(e) } From 13a600b2bd8a992fe8a8e1537066ed4b97af19bc Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 28 Mar 2025 16:31:13 -0700 Subject: [PATCH 23/39] cleanup, add tests --- src/client/rpc.rs | 9 ---- src/main.rs | 5 --- src/proxy.rs | 89 +++++++++++++++++---------------------- src/server.rs | 104 +++++++++++++++++++++++++++++++++++----------- 4 files changed, 117 insertions(+), 90 deletions(-) diff --git a/src/client/rpc.rs b/src/client/rpc.rs index c02dbf8a..ff9b31c6 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -7,15 +7,12 @@ use alloy_rpc_types_engine::{ }; use clap::{Parser, arg}; use http::Uri; -use jsonrpsee::core::client::ClientT; use jsonrpsee::http_client::transport::HttpBackend; use jsonrpsee::http_client::{HttpClient, HttpClientBuilder}; -use jsonrpsee::rpc_params; use jsonrpsee::types::ErrorObjectOwned; use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelopeV3, OpPayloadAttributes}; use opentelemetry::trace::SpanKind; use paste::paste; -use serde_json::Value; use std::path::PathBuf; use std::time::Duration; use thiserror::Error; @@ -220,12 +217,6 @@ impl RpcClient { Ok(res) } - - pub async fn health(&self) -> ClientResult<()> { - let method = "web3_clientVersion"; - let _res: Value = self.auth_client.request(method, rpc_params![]).await?; - Ok(()) - } } /// Generates Clap argument structs with a prefix to create a unique namespace when specifing RPC client config via the CLI. diff --git a/src/main.rs b/src/main.rs index cbfa25ab..6fe558f9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -237,11 +237,6 @@ async fn main() -> eyre::Result<()> { .await?; let handle = server.start(module); - while l2_client.health().await.is_err() { - info!("waiting for l2 client to be healthy"); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - } - let stop_handle = handle.clone(); // Capture SIGINT and SIGTERM diff --git a/src/proxy.rs b/src/proxy.rs index 0a192c57..72a31db0 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -1,13 +1,13 @@ use crate::client::http::HttpClient; use crate::server::PayloadSource; use alloy_rpc_types_engine::JwtSecret; -use futures::FutureExt as _; use http::Uri; use jsonrpsee::core::{BoxError, http_helpers}; use jsonrpsee::http_client::{HttpBody, HttpRequest, HttpResponse}; use std::task::{Context, Poll}; use std::{future::Future, pin::Pin}; use tower::{Layer, Service}; +use tracing::debug; const MULTIPLEX_METHODS: [&str; 4] = [ "engine_", @@ -109,50 +109,38 @@ where let mut service = self.clone(); service.inner = std::mem::replace(&mut self.inner, service.inner); - async move { - let res = async move { - let (parts, body) = req.into_parts(); - let (body_bytes, _) = - http_helpers::read_body(&parts.headers, body, u32::MAX).await?; - - // Deserialize the bytes to find the method - let method = serde_json::from_slice::(&body_bytes)? - .method - .to_string(); - - if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { - if FORWARD_REQUESTS.contains(&method.as_str()) { - let builder_req = HttpRequest::from_parts( - parts.clone(), - HttpBody::from(body_bytes.clone()), - ); - let builder_method = method.clone(); - let mut builder_client = service.builder_client.clone(); - tokio::spawn(async move { - let _ = builder_client.forward(builder_req, builder_method).await; - }); - - let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - service.l2_client.forward(l2_req, method).await - } else { - let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - service.inner.call(req).await.map_err(|e| e.into()) - } + let fut = async move { + let (parts, body) = req.into_parts(); + let (body_bytes, _) = http_helpers::read_body(&parts.headers, body, u32::MAX).await?; + + // Deserialize the bytes to find the method + let method = serde_json::from_slice::(&body_bytes)? + .method + .to_string(); + + if MULTIPLEX_METHODS.iter().any(|&m| method.starts_with(m)) { + if FORWARD_REQUESTS.contains(&method.as_str()) { + let builder_req = + HttpRequest::from_parts(parts.clone(), HttpBody::from(body_bytes.clone())); + let builder_method = method.clone(); + let mut builder_client = service.builder_client.clone(); + tokio::spawn(async move { + let _ = builder_client.forward(builder_req, builder_method).await; + }); + + let l2_req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + service.l2_client.forward(l2_req, method).await } else { let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); - service.l2_client.forward(req, method).await + debug!(target: "proxy::call", message = "proxying request to rollup-boost server", ?method); + service.inner.call(req).await.map_err(|e| e.into()) } + } else { + let req = HttpRequest::from_parts(parts, HttpBody::from(body_bytes)); + service.l2_client.forward(req, method).await } - .await; - - Ok(res.unwrap_or_else(|e| { - HttpResponse::builder() - .status(500) - .body(HttpBody::from(e.to_string())) - .unwrap() - })) - } - .boxed() + }; + Box::pin(fut) } } @@ -164,12 +152,14 @@ mod tests { use alloy_primitives::{B256, Bytes, U64, U128, hex}; use alloy_rpc_types_engine::JwtSecret; use alloy_rpc_types_eth::erc4337::TransactionConditional; + use http::StatusCode; use http_body_util::BodyExt; use hyper::service::service_fn; use hyper_util::client::legacy::Client; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::rt::{TokioExecutor, TokioIo}; use jsonrpsee::server::Server; + use jsonrpsee::types::{ErrorCode, ErrorObject}; use jsonrpsee::{ RpcModule, core::{ClientError, client::ClientT}, @@ -386,6 +376,11 @@ mod tests { async fn proxy_failure() { let response = send_request("non_existent_method").await; assert!(response.is_err()); + let expected_error = ErrorObject::from(ErrorCode::MethodNotFound).into_owned(); + assert!(matches!( + response.unwrap_err(), + ClientError::Call(e) if e == expected_error + )); } async fn does_not_proxy_engine_method() { @@ -404,16 +399,8 @@ mod tests { let health_check_url = format!("http://{ADDR}:{PORT}/healthz"); let health_response = client.get(health_check_url.parse::().unwrap()).await; assert!(health_response.is_ok()); - let b = health_response - .unwrap() - .into_body() - .collect() - .await - .unwrap() - .to_bytes(); - // Convert the collected bytes to a string - let body_string = String::from_utf8(b.to_vec()).unwrap(); - assert_eq!(body_string, "OK"); + let status = health_response.unwrap().status(); + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); proxy_server.stop().unwrap(); proxy_server.stopped().await; diff --git a/src/server.rs b/src/server.rs index 5b20034a..3c5f4d70 100644 --- a/src/server.rs +++ b/src/server.rs @@ -440,6 +440,7 @@ impl EngineApiServer for RollupBoostServer { #[allow(clippy::complexity)] mod tests { use crate::probe::ProbeLayer; + use crate::proxy::ProxyLayer; use super::*; use alloy_primitives::hex; @@ -449,10 +450,10 @@ mod tests { }; use alloy_rpc_types_engine::JwtSecret; - use http::Uri; + use http::{StatusCode, Uri}; use jsonrpsee::RpcModule; use jsonrpsee::http_client::HttpClient; - use jsonrpsee::server::{ServerBuilder, ServerHandle}; + use jsonrpsee::server::{Server, ServerBuilder, ServerHandle}; use parking_lot::Mutex; use std::net::SocketAddr; use std::str::FromStr; @@ -529,8 +530,9 @@ mod tests { l2_mock: MockEngineServer, builder_server: ServerHandle, builder_mock: MockEngineServer, - proxy_server: ServerHandle, - client: HttpClient, + server: ServerHandle, + rpc_client: HttpClient, + http_client: reqwest::Client, } impl TestHarness { @@ -543,16 +545,21 @@ mod tests { let l2_auth_rpc = Uri::from_str(&format!("http://{}:{}", HOST, L2_PORT)).unwrap(); let l2_client = - RpcClient::new(l2_auth_rpc, jwt_secret, 2000, PayloadSource::L2).unwrap(); + RpcClient::new(l2_auth_rpc.clone(), jwt_secret, 2000, PayloadSource::L2).unwrap(); let builder_auth_rpc = Uri::from_str(&format!("http://{}:{}", HOST, BUILDER_PORT)).unwrap(); - let builder_client = - RpcClient::new(builder_auth_rpc, jwt_secret, 2000, PayloadSource::Builder).unwrap(); + let builder_client = RpcClient::new( + builder_auth_rpc.clone(), + jwt_secret, + 2000, + PayloadSource::Builder, + ) + .unwrap(); - let (_probe_layer, probes) = ProbeLayer::new(); + let (probe_layer, probes) = ProbeLayer::new(); - let rollup_boost_client = RollupBoostServer::new( + let rollup_boost = RollupBoostServer::new( l2_client, builder_client, boost_sync, @@ -560,36 +567,65 @@ mod tests { probes, ); - let module: RpcModule<()> = rollup_boost_client.try_into().unwrap(); + let module: RpcModule<()> = rollup_boost.try_into().unwrap(); - let proxy_server = ServerBuilder::default() + let http_middleware = + tower::ServiceBuilder::new() + .layer(probe_layer) + .layer(ProxyLayer::new( + l2_auth_rpc, + jwt_secret, + builder_auth_rpc, + jwt_secret, + )); + + let server = Server::builder() + .set_http_middleware(http_middleware) .build("0.0.0.0:8556".parse::().unwrap()) .await .unwrap() .start(module); + + // let proxy_server = ServerBuilder::default() + // .build("0.0.0.0:8556".parse::().unwrap()) + // .await + // .unwrap() + // .start(module); let l2_mock = l2_mock.unwrap_or(MockEngineServer::new()); let builder_mock = builder_mock.unwrap_or(MockEngineServer::new()); let l2_server = spawn_server(l2_mock.clone(), L2_ADDR).await; let builder_server = spawn_server(builder_mock.clone(), BUILDER_ADDR).await; + let rpc_client = HttpClient::builder() + .build(format!("http://{SERVER_ADDR}")) + .unwrap(); + let http_client = reqwest::Client::new(); + TestHarness { l2_server, l2_mock, builder_server, builder_mock, - proxy_server, - client: HttpClient::builder() - .build(format!("http://{SERVER_ADDR}")) - .unwrap(), + server, + rpc_client, + http_client, } } + async fn get(&self, path: &str) -> reqwest::Response { + self.http_client + .get(format!("http://{}/{}", SERVER_ADDR, path)) + .send() + .await + .unwrap() + } + async fn cleanup(self) { self.l2_server.stop().unwrap(); self.l2_server.stopped().await; self.builder_server.stop().unwrap(); self.builder_server.stopped().await; - self.proxy_server.stop().unwrap(); - self.proxy_server.stopped().await; + self.server.stop().unwrap(); + self.server.stopped().await; } } @@ -604,13 +640,20 @@ mod tests { async fn engine_success() { let test_harness = TestHarness::new(false, None, None).await; + // Since no blocks have been created, the service should be unavailable + let health = test_harness.get("healthz").await; + assert_eq!(health.status(), StatusCode::SERVICE_UNAVAILABLE); + // test fork_choice_updated_v3 success let fcu = ForkchoiceState { head_block_hash: FixedBytes::random(), safe_block_hash: FixedBytes::random(), finalized_block_hash: FixedBytes::random(), }; - let fcu_response = test_harness.client.fork_choice_updated_v3(fcu, None).await; + let fcu_response = test_harness + .rpc_client + .fork_choice_updated_v3(fcu, None) + .await; assert!(fcu_response.is_ok()); let fcu_requests = test_harness.l2_mock.fcu_requests.clone(); { @@ -627,7 +670,7 @@ mod tests { // test new_payload_v3 success let new_payload_response = test_harness - .client + .rpc_client .new_payload_v3( test_harness .l2_mock @@ -667,7 +710,7 @@ mod tests { // test get_payload_v3 success let get_payload_response = test_harness - .client + .rpc_client .get_payload_v3(PayloadId::new([0, 0, 0, 0, 0, 0, 0, 1])) .await; assert!(get_payload_response.is_ok()); @@ -686,6 +729,11 @@ mod tests { assert_eq!(*req, PayloadId::new([0, 0, 0, 0, 0, 0, 0, 1])); } + // Now that a block has been produced by the l2 but not the builder + // the health status should be Partial Content + let health = test_harness.get("healthz").await; + assert_eq!(health.status(), StatusCode::PARTIAL_CONTENT); + test_harness.cleanup().await; } @@ -697,7 +745,10 @@ mod tests { safe_block_hash: FixedBytes::random(), finalized_block_hash: FixedBytes::random(), }; - let fcu_response = test_harness.client.fork_choice_updated_v3(fcu, None).await; + let fcu_response = test_harness + .rpc_client + .fork_choice_updated_v3(fcu, None) + .await; assert!(fcu_response.is_ok()); sleep(std::time::Duration::from_millis(100)).await; @@ -713,7 +764,7 @@ mod tests { // test new_payload_v3 success let new_payload_response = test_harness - .client + .rpc_client .new_payload_v3( test_harness .l2_mock @@ -756,7 +807,7 @@ mod tests { // test get_payload_v3 return l2 payload if builder payload is invalid let get_payload_response = test_harness - .client + .rpc_client .get_payload_v3(PayloadId::new([0, 0, 0, 0, 0, 0, 0, 0])) .await; assert!(get_payload_response.is_ok()); @@ -830,7 +881,10 @@ mod tests { safe_block_hash: FixedBytes::random(), finalized_block_hash: FixedBytes::random(), }; - let fcu_response = test_harness.client.fork_choice_updated_v3(fcu, None).await; + let fcu_response = test_harness + .rpc_client + .fork_choice_updated_v3(fcu, None) + .await; assert!(fcu_response.is_ok()); // wait for builder to observe the FCU call @@ -843,7 +897,7 @@ mod tests { } // Test getPayload call - let get_res = test_harness.client.get_payload_v3(same_id).await; + let get_res = test_harness.rpc_client.get_payload_v3(same_id).await; assert!(get_res.is_ok()); // wait for builder to observe the getPayload call From 98adf3a959e9a43250627baf7ca104578309b2f2 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 28 Mar 2025 16:42:47 -0700 Subject: [PATCH 24/39] remove stray comment --- src/server.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/server.rs b/src/server.rs index 3c5f4d70..e2910d9d 100644 --- a/src/server.rs +++ b/src/server.rs @@ -586,11 +586,6 @@ mod tests { .unwrap() .start(module); - // let proxy_server = ServerBuilder::default() - // .build("0.0.0.0:8556".parse::().unwrap()) - // .await - // .unwrap() - // .start(module); let l2_mock = l2_mock.unwrap_or(MockEngineServer::new()); let builder_mock = builder_mock.unwrap_or(MockEngineServer::new()); let l2_server = spawn_server(l2_mock.clone(), L2_ADDR).await; From 8ba95f2074076bca155b97d769afcd633543ab81 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Tue, 8 Apr 2025 15:44:34 -0700 Subject: [PATCH 25/39] chore: rm mocks --- tests/mock.rs | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/mock.rs diff --git a/tests/mock.rs b/tests/mock.rs deleted file mode 100644 index 8b137891..00000000 --- a/tests/mock.rs +++ /dev/null @@ -1 +0,0 @@ - From 2a5c6133d1795d002a315a3e738acdacbf354b83 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Tue, 8 Apr 2025 15:53:20 -0700 Subject: [PATCH 26/39] chore: fmt --- src/server.rs | 2 +- tests/common/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.rs b/src/server.rs index a353ea1c..6197ae51 100644 --- a/src/server.rs +++ b/src/server.rs @@ -625,10 +625,10 @@ mod tests { use super::*; use alloy_primitives::hex; use alloy_primitives::{FixedBytes, U256}; + use alloy_rpc_types_engine::JwtSecret; use alloy_rpc_types_engine::{ BlobsBundleV1, ExecutionPayloadV1, ExecutionPayloadV2, PayloadStatusEnum, }; - use alloy_rpc_types_engine::JwtSecret; use http::{StatusCode, Uri}; use jsonrpsee::RpcModule; use jsonrpsee::http_client::HttpClient; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 10369b03..5fd97c96 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,7 +6,6 @@ use alloy_rpc_types_engine::{ ForkchoiceState, ForkchoiceUpdated, PayloadAttributes, PayloadId, PayloadStatus, PayloadStatusEnum, }; -use tower_http::sensitive_headers::SetSensitiveRequestHeaders; use alloy_rpc_types_eth::BlockNumberOrTag; use bytes::BytesMut; use futures::FutureExt; @@ -37,6 +36,7 @@ use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, ImageExt}; use time::{OffsetDateTime, format_description}; use tokio::io::AsyncWriteExt as _; +use tower_http::sensitive_headers::SetSensitiveRequestHeaders; use tracing::info; /// Default JWT token for testing purposes From 3b06bd96d60c3b5d6e4c28bfdecb4277cbfe76df Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Tue, 22 Apr 2025 15:50:31 -0700 Subject: [PATCH 27/39] fix: default to healthy status --- scripts/ci/kurtosis-params.yaml | 1 - src/probe.rs | 2 +- src/proxy.rs | 2 +- src/server.rs | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/ci/kurtosis-params.yaml b/scripts/ci/kurtosis-params.yaml index 6a0678b3..873b8991 100644 --- a/scripts/ci/kurtosis-params.yaml +++ b/scripts/ci/kurtosis-params.yaml @@ -14,7 +14,6 @@ optimism_package: fjord_time_offset: 0 granite_time_offset: 0 isthmus_time_offset: 5 - fund_dev_accounts: true mev_params: rollup_boost_image: "flashbots/rollup-boost:develop" additional_services: diff --git a/src/probe.rs b/src/probe.rs index 645712f9..793ae10f 100644 --- a/src/probe.rs +++ b/src/probe.rs @@ -15,13 +15,13 @@ use tower::{Layer, Service}; #[derive(Copy, Clone, Debug, Default)] pub enum Health { /// Indicates that the builder is building blocks + #[default] Healthy, /// Indicates that the l2 is building blocks, but the builder is not PartialContent, /// Indicates that blocks are not being built by either the l2 or the builder /// /// Service starts out unavailable until the first blocks are built - #[default] ServiceUnavailable, } diff --git a/src/proxy.rs b/src/proxy.rs index c827a0ba..7daff475 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -400,7 +400,7 @@ mod tests { let health_response = client.get(health_check_url.parse::().unwrap()).await; assert!(health_response.is_ok()); let status = health_response.unwrap().status(); - assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(status, StatusCode::OK); proxy_server.stop().unwrap(); proxy_server.stopped().await; diff --git a/src/server.rs b/src/server.rs index 6197ae51..402f7f48 100644 --- a/src/server.rs +++ b/src/server.rs @@ -816,7 +816,7 @@ mod tests { // Since no blocks have been created, the service should be unavailable let health = test_harness.get("healthz").await; - assert_eq!(health.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(health.status(), StatusCode::OK); // test fork_choice_updated_v3 success let fcu = ForkchoiceState { From 6d8b40ae739ab616280059e0c5153e37b056d4f8 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Wed, 23 Apr 2025 13:59:38 -0700 Subject: [PATCH 28/39] chore: fix dockerignore --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 23acc428..3b2f216e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -24,3 +24,4 @@ docs/ # Scripts that aren't needed for runtime scripts/ tests/ +**/integration \ No newline at end of file From a6a94ef069b3a707dd60fe6ce41c355769833782 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Thu, 24 Apr 2025 12:45:33 -0700 Subject: [PATCH 29/39] feat: add background process to query block height as a health check for non-sequencing el's --- src/cli.rs | 5 +++++ src/client/rpc.rs | 6 ++++++ src/health.rs | 53 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 +++ src/server.rs | 42 +++++++++++++++++++++++++++++-------- 5 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 src/health.rs diff --git a/src/cli.rs b/src/cli.rs index db9b62f3..7447c59c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -27,6 +27,10 @@ pub struct Args { #[clap(flatten)] pub l2_client: L2ClientArgs, + /// Duration in seconds between async health checks on the builder, and the l2 client + #[arg(long, env, default_value = "60")] + pub health_check_interval: u64, + /// Disable using the proposer to sync the builder node #[arg(long, env, default_value = "false")] pub no_boost_sync: bool, @@ -164,6 +168,7 @@ impl Args { boost_sync_enabled, self.execution_mode, probes, + self.health_check_interval, ); // Spawn the debug server diff --git a/src/client/rpc.rs b/src/client/rpc.rs index ea3a610d..970e3ba8 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -322,6 +322,12 @@ impl RpcClient { } } } + + pub async fn get_block_number(&self) -> ClientResult { + info!("Sending get_block_number to {}", self.payload_source); + let block_number = self.auth_client.get_block_number().await.set_code()?; + Ok(block_number) + } } /// Generates Clap argument structs with a prefix to create a unique namespace when specifying RPC client config via the CLI. diff --git a/src/health.rs b/src/health.rs new file mode 100644 index 00000000..e618b207 --- /dev/null +++ b/src/health.rs @@ -0,0 +1,53 @@ +use std::{sync::Arc, time::Duration}; + +use tokio::{ + task::JoinHandle, + time::{Instant, sleep_until}, +}; +use tracing::{error, info}; + +use crate::{Health, Probes, RpcClient}; + +pub struct HealthHandle { + pub probes: Arc, + pub builder_client: Arc, + pub l2_client: Arc, + pub health_check_interval: u64, +} + +impl HealthHandle { + pub fn spawn(self) -> JoinHandle<()> { + let handle = tokio::spawn(async move { + loop { + let (l2_block_height, builder_block_height) = tokio::join!( + self.l2_client.get_block_number(), + self.builder_client.get_block_number() + ); + match l2_block_height { + Err(e) => { + error!(target: "rollup_boost::health", "Failed to get block height from l2 client: {}", e); + self.probes.set_health(Health::ServiceUnavailable); + } + Ok(l2_block_height) => match builder_block_height { + Err(e) => { + error!(target: "rollup_boost::health", "Failed to get block height from builder client: {}", e); + self.probes.set_health(Health::PartialContent); + } + Ok(builder_block_height) => { + if builder_block_height != l2_block_height { + error!(target: "rollup_boost::health", "Builder and L2 client block heights do not match: builder: {}, l2: {}", builder_block_height, l2_block_height); + self.probes.set_health(Health::PartialContent); + } else { + info!(target: "rollup_boost::health", %builder_block_height, "Health Status Check Passed - Builder and L2 client block heights match"); + self.probes.set_health(Health::Healthy); + } + } + }, + } + sleep_until(Instant::now() + Duration::from_secs(self.health_check_interval)).await; + } + }); + + handle + } +} diff --git a/src/lib.rs b/src/lib.rs index 2cf2b9de..37e4ecfa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,3 +24,6 @@ pub use tracing::*; mod probe; pub use probe::*; + +mod health; +pub use health::*; diff --git a/src/server.rs b/src/server.rs index 91d3371f..775e3e57 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,3 +1,4 @@ +use crate::HealthHandle; use crate::client::rpc::RpcClient; use crate::debug_api::DebugServer; use crate::probe::{Health, Probes}; @@ -21,7 +22,7 @@ use op_alloy_rpc_types_engine::{ OpPayloadAttributes, }; use serde::{Deserialize, Serialize}; - +use tokio::task::JoinHandle; use tracing::{debug, info, instrument}; use jsonrpsee::proc_macros::rpc; @@ -123,12 +124,12 @@ impl ExecutionMode { } } -#[derive(Clone)] pub struct RollupBoostServer { pub l2_client: Arc, pub builder_client: Arc, pub boost_sync: bool, pub payload_trace_context: Arc, + health_handle: JoinHandle<()>, execution_mode: Arc>, probes: Arc, } @@ -140,7 +141,16 @@ impl RollupBoostServer { boost_sync: bool, initial_execution_mode: ExecutionMode, probes: Arc, + health_check_interval: u64, ) -> Self { + let health_handle = HealthHandle { + probes: probes.clone(), + builder_client: Arc::new(builder_client.clone()), + l2_client: Arc::new(l2_client.clone()), + health_check_interval, + } + .spawn(); + // Spawn a thread for the continuous health check on the builder Self { l2_client: Arc::new(l2_client), builder_client: Arc::new(builder_client), @@ -148,6 +158,7 @@ impl RollupBoostServer { payload_trace_context: Arc::new(PayloadTraceContext::new()), execution_mode: Arc::new(Mutex::new(initial_execution_mode)), probes, + health_handle, } } @@ -160,6 +171,10 @@ impl RollupBoostServer { pub fn execution_mode(&self) -> ExecutionMode { *self.execution_mode.lock() } + + pub fn health_handle(&self) -> &JoinHandle<()> { + &self.health_handle + } } impl TryInto> for RollupBoostServer { @@ -167,7 +182,7 @@ impl TryInto> for RollupBoostServer { fn try_into(self) -> Result, Self::Error> { let mut module: RpcModule<()> = RpcModule::new(()); - module.merge(EngineApiServer::into_rpc(self.clone()))?; + module.merge(EngineApiServer::into_rpc(self))?; for method in module.method_names() { info!(?method, "method registered"); @@ -203,22 +218,22 @@ impl PayloadSource { } } -#[rpc(server, client, namespace = "engine")] +#[rpc(server, client)] pub trait EngineApi { - #[method(name = "forkchoiceUpdatedV3")] + #[method(name = "engine_forkchoiceUpdatedV3")] async fn fork_choice_updated_v3( &self, fork_choice_state: ForkchoiceState, payload_attributes: Option, ) -> RpcResult; - #[method(name = "getPayloadV3")] + #[method(name = "engine_getPayloadV3")] async fn get_payload_v3( &self, payload_id: PayloadId, ) -> RpcResult; - #[method(name = "newPayloadV3")] + #[method(name = "engine_newPayloadV3")] async fn new_payload_v3( &self, payload: ExecutionPayloadV3, @@ -226,13 +241,13 @@ pub trait EngineApi { parent_beacon_block_root: B256, ) -> RpcResult; - #[method(name = "getPayloadV4")] + #[method(name = "engine_getPayloadV4")] async fn get_payload_v4( &self, payload_id: PayloadId, ) -> RpcResult; - #[method(name = "newPayloadV4")] + #[method(name = "engine_newPayloadV4")] async fn new_payload_v4( &self, payload: OpExecutionPayloadV4, @@ -240,6 +255,9 @@ pub trait EngineApi { parent_beacon_block_root: B256, execution_requests: Vec, ) -> RpcResult; + + #[method(name = "eth_getBlockNumber")] + async fn get_block_number(&self) -> RpcResult; } #[async_trait] @@ -410,6 +428,11 @@ impl EngineApiServer for RollupBoostServer { })) .await } + + async fn get_block_number(&self) -> RpcResult { + info!("received get_block_number"); + Ok(self.l2_client.get_block_number().await?) + } } #[derive(Debug, Clone)] @@ -759,6 +782,7 @@ mod tests { boost_sync, ExecutionMode::Enabled, probes, + 10, ); let module: RpcModule<()> = rollup_boost.try_into().unwrap(); From bb07cdf6921f66b22b62666c4f8eed9556938f09 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Thu, 24 Apr 2025 13:10:44 -0700 Subject: [PATCH 30/39] fix: signatures --- src/client/rpc.rs | 7 +++---- src/health.rs | 4 ++-- src/server.rs | 11 +++++------ 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/client/rpc.rs b/src/client/rpc.rs index 970e3ba8..c99a3319 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -2,7 +2,7 @@ use crate::client::auth::AuthLayer; use crate::server::{ EngineApiClient, NewPayload, OpExecutionPayloadEnvelope, PayloadSource, Version, }; -use alloy_primitives::{B256, Bytes}; +use alloy_primitives::{B256, Bytes, U256}; use alloy_rpc_types_engine::{ ExecutionPayload, ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, JwtError, JwtSecret, PayloadId, PayloadStatus, @@ -323,9 +323,8 @@ impl RpcClient { } } - pub async fn get_block_number(&self) -> ClientResult { - info!("Sending get_block_number to {}", self.payload_source); - let block_number = self.auth_client.get_block_number().await.set_code()?; + pub async fn block_number(&self) -> ClientResult { + let block_number = self.auth_client.block_number().await.set_code()?; Ok(block_number) } } diff --git a/src/health.rs b/src/health.rs index e618b207..69d7dd30 100644 --- a/src/health.rs +++ b/src/health.rs @@ -20,8 +20,8 @@ impl HealthHandle { let handle = tokio::spawn(async move { loop { let (l2_block_height, builder_block_height) = tokio::join!( - self.l2_client.get_block_number(), - self.builder_client.get_block_number() + self.l2_client.block_number(), + self.builder_client.block_number() ); match l2_block_height { Err(e) => { diff --git a/src/server.rs b/src/server.rs index 775e3e57..8907667e 100644 --- a/src/server.rs +++ b/src/server.rs @@ -2,7 +2,7 @@ use crate::HealthHandle; use crate::client::rpc::RpcClient; use crate::debug_api::DebugServer; use crate::probe::{Health, Probes}; -use alloy_primitives::{B256, Bytes}; +use alloy_primitives::{B256, Bytes, U256}; use metrics::counter; use moka::sync::Cache; use opentelemetry::trace::SpanKind; @@ -256,8 +256,8 @@ pub trait EngineApi { execution_requests: Vec, ) -> RpcResult; - #[method(name = "eth_getBlockNumber")] - async fn get_block_number(&self) -> RpcResult; + #[method(name = "eth_blockNumber")] + async fn block_number(&self) -> RpcResult; } #[async_trait] @@ -429,9 +429,8 @@ impl EngineApiServer for RollupBoostServer { .await } - async fn get_block_number(&self) -> RpcResult { - info!("received get_block_number"); - Ok(self.l2_client.get_block_number().await?) + async fn block_number(&self) -> RpcResult { + Ok(self.l2_client.block_number().await?) } } From c6855a4e01275f9b659853f434edb404874ada91 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Thu, 24 Apr 2025 13:12:47 -0700 Subject: [PATCH 31/39] chore: update comments --- src/server.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index 8907667e..8dd87ec7 100644 --- a/src/server.rs +++ b/src/server.rs @@ -143,6 +143,7 @@ impl RollupBoostServer { probes: Arc, health_check_interval: u64, ) -> Self { + // Spawns a helth check service in the background to continuously check the health of the L2 and builder clients let health_handle = HealthHandle { probes: probes.clone(), builder_client: Arc::new(builder_client.clone()), @@ -150,7 +151,7 @@ impl RollupBoostServer { health_check_interval, } .spawn(); - // Spawn a thread for the continuous health check on the builder + Self { l2_client: Arc::new(l2_client), builder_client: Arc::new(builder_client), From eae3273238cb347a3598b89ff246853c86cac38e Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Thu, 24 Apr 2025 13:21:51 -0700 Subject: [PATCH 32/39] chore: clippy --- src/health.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/health.rs b/src/health.rs index 69d7dd30..cf87f50d 100644 --- a/src/health.rs +++ b/src/health.rs @@ -17,7 +17,7 @@ pub struct HealthHandle { impl HealthHandle { pub fn spawn(self) -> JoinHandle<()> { - let handle = tokio::spawn(async move { + tokio::spawn(async move { loop { let (l2_block_height, builder_block_height) = tokio::join!( self.l2_client.block_number(), @@ -46,8 +46,6 @@ impl HealthHandle { } sleep_until(Instant::now() + Duration::from_secs(self.health_check_interval)).await; } - }); - - handle + }) } } From 200b4074864c10ac26385dc499ee92d79d2a43f7 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Thu, 24 Apr 2025 16:05:56 -0700 Subject: [PATCH 33/39] test: add tests --- src/health.rs | 227 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) diff --git a/src/health.rs b/src/health.rs index cf87f50d..99b86e10 100644 --- a/src/health.rs +++ b/src/health.rs @@ -49,3 +49,230 @@ impl HealthHandle { }) } } + +#[cfg(test)] +mod tests { + use std::net::SocketAddr; + + use alloy_primitives::U256; + use http::Uri; + use http_body_util::BodyExt; + use hyper::service::service_fn; + use hyper_util::rt::TokioIo; + use reth_rpc_layer::JwtSecret; + use serde_json::json; + use tokio::net::TcpListener; + + use super::*; + use crate::{PayloadSource, Probes}; + + pub struct MockHttpServer { + addr: SocketAddr, + join_handle: JoinHandle<()>, + } + + impl Drop for MockHttpServer { + fn drop(&mut self) { + self.join_handle.abort(); + } + } + + impl MockHttpServer { + async fn serve(f: fn(hyper::Request) -> S) -> eyre::Result + where + S: Future, hyper::Error>> + + Send + + Sync + + 'static, + { + { + let listener = TcpListener::bind("0.0.0.0:0").await?; + let addr = listener.local_addr()?; + + let handle = tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => { + let io = TokioIo::new(stream); + tokio::spawn(async move { + if let Err(err) = hyper::server::conn::http1::Builder::new() + .serve_connection(io, service_fn(move |req| f(req))) + .await + { + eprintln!("Error serving connection: {}", err); + } + }); + } + Err(e) => eprintln!("Error accepting connection: {}", e), + } + } + }); + + Ok(Self { + addr, + join_handle: handle, + }) + } + } + } + + async fn handler_0( + req: hyper::Request, + ) -> Result, hyper::Error> { + let body_bytes = match req.into_body().collect().await { + Ok(buf) => buf.to_bytes(), + Err(_) => { + let error_response = json!({ + "jsonrpc": "2.0", + "error": { "code": -32700, "message": "Failed to read request body" }, + "id": null + }); + return Ok(hyper::Response::new(error_response.to_string())); + } + }; + + let request_body: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(json) => json, + Err(_) => { + let error_response = json!({ + "jsonrpc": "2.0", + "error": { "code": -32700, "message": "Invalid JSON format" }, + "id": null + }); + return Ok(hyper::Response::new(error_response.to_string())); + } + }; + + let method = request_body["method"].as_str().unwrap_or_default(); + + let response = match method { + "eth_blockNumber" => json!({ + "jsonrpc": "2.0", + "result": format!("{}", U256::ZERO), + "id": request_body["id"] + }), + _ => { + let error_response = json!({ + "jsonrpc": "2.0", + "error": { "code": -32601, "message": "Method not found" }, + "id": request_body["id"] + }); + return Ok(hyper::Response::new(error_response.to_string())); + } + }; + + Ok(hyper::Response::new(response.to_string())) + } + + async fn handler_1( + req: hyper::Request, + ) -> Result, hyper::Error> { + let body_bytes = match req.into_body().collect().await { + Ok(buf) => buf.to_bytes(), + Err(_) => { + let error_response = json!({ + "jsonrpc": "2.0", + "error": { "code": -32700, "message": "Failed to read request body" }, + "id": null + }); + return Ok(hyper::Response::new(error_response.to_string())); + } + }; + + let request_body: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(json) => json, + Err(_) => { + let error_response = json!({ + "jsonrpc": "2.0", + "error": { "code": -32700, "message": "Invalid JSON format" }, + "id": null + }); + return Ok(hyper::Response::new(error_response.to_string())); + } + }; + + let method = request_body["method"].as_str().unwrap_or_default(); + + let response = match method { + "eth_blockNumber" => json!({ + "jsonrpc": "2.0", + "result": format!("{}", U256::from(1)), + "id": request_body["id"] + }), + _ => { + let error_response = json!({ + "jsonrpc": "2.0", + "error": { "code": -32601, "message": "Method not found" }, + "id": request_body["id"] + }); + return Ok(hyper::Response::new(error_response.to_string())); + } + }; + + Ok(hyper::Response::new(response.to_string())) + } + + #[tokio::test] + async fn test_health_check_healthy() -> eyre::Result<()> { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + let probes = Arc::new(Probes::default()); + let builder = MockHttpServer::serve(handler_0).await.unwrap(); + let l2 = MockHttpServer::serve(handler_0).await.unwrap(); + + let builder_client = Arc::new(RpcClient::new( + format!("http://{}", builder.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::Builder, + )?); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); + let health_handle = HealthHandle { + probes: probes.clone(), + builder_client: builder_client.clone(), + l2_client: l2_client.clone(), + health_check_interval: 60, + }; + + let _ = health_handle.spawn(); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(matches!(probes.health(), Health::Healthy)); + Ok(()) + } + + #[tokio::test] + async fn test_health_check_partial_content() -> eyre::Result<()> { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + let probes = Arc::new(Probes::default()); + let builder = MockHttpServer::serve(handler_0).await.unwrap(); + let l2 = MockHttpServer::serve(handler_1).await.unwrap(); + + let builder_client = Arc::new(RpcClient::new( + format!("http://{}", builder.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::Builder, + )?); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); + let health_handle = HealthHandle { + probes: probes.clone(), + builder_client: builder_client.clone(), + l2_client: l2_client.clone(), + health_check_interval: 60, + }; + + let _ = health_handle.spawn(); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(matches!(probes.health(), Health::PartialContent)); + Ok(()) + } +} From c4d0c769fe280e7d8f4cf6585b6495030516005a Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Thu, 24 Apr 2025 16:32:39 -0700 Subject: [PATCH 34/39] fix: stress tests --- scripts/ci/stress.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/stress.sh b/scripts/ci/stress.sh index d6cec2dc..9543ae33 100755 --- a/scripts/ci/stress.sh +++ b/scripts/ci/stress.sh @@ -32,8 +32,8 @@ run() { # the transactions will be included in the canonical blocks and finalized. # Figure out first the builder's JSON-RPC URL - ROLLUP_BOOST_SOCKET=$(kurtosis port print op-rollup-boost op-rollup-boost-1-op-kurtosis rpc) - OP_RETH_BUILDER_SOCKET=$(kurtosis port print op-rollup-boost op-el-builder-1-op-reth-op-node-op-kurtosis rpc) + ROLLUP_BOOST_SOCKET=$(kurtosis port print op-rollup-boost op-rollup-boost-2151908-1-op-kurtosis rpc) + OP_RETH_BUILDER_SOCKET=$(kurtosis port print op-rollup-boost op-el-builder-2151908-1-op-reth-op-node-op-kurtosis rpc) # Private key with prefunded balance PREFUNDED_PRIV_KEY=0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d From f15e7c4c05915cb123a304b7042776655284915e Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 25 Apr 2025 15:26:37 -0700 Subject: [PATCH 35/39] fix: change health check to check unsafe head progression on builder --- Cargo.lock | 1 + Cargo.toml | 3 +- src/cli.rs | 5 ++ src/client/rpc.rs | 17 ++++- src/health.rs | 183 +++++++++++++++++++++++----------------------- src/server.rs | 17 +++-- 6 files changed, 121 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50dc06e5..4a688ec4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3775,6 +3775,7 @@ dependencies = [ name = "rollup-boost" version = "0.1.0" dependencies = [ + "alloy-consensus", "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", diff --git a/Cargo.toml b/Cargo.toml index 7784c333..2df4d56b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] op-alloy-rpc-types-engine = "0.12.0" alloy-rpc-types-engine = "0.13.0" +alloy-rpc-types-eth = "0.13.0" alloy-primitives = { version = "0.8.10", features = ["rand"] } tokio = { version = "1", features = ["full"] } tracing = "0.1.4" @@ -51,7 +52,7 @@ rand = "0.9.0" time = { version = "0.3.36", features = ["macros", "formatting", "parsing"] } op-alloy-consensus = "0.12.0" alloy-eips = { version = "0.13.0", features = ["serde"] } -alloy-rpc-types-eth = "0.13.0" +alloy-consensus = {version = "0.13.0", features = ["serde"] } anyhow = "1.0" testcontainers = { version = "0.23.3" } assert_cmd = "2.0.10" diff --git a/src/cli.rs b/src/cli.rs index 7447c59c..d0d58d3f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -31,6 +31,10 @@ pub struct Args { #[arg(long, env, default_value = "60")] pub health_check_interval: u64, + /// Max duration in seconds between the unsafe head block and the current time to be considered healthy + #[arg(long, env, default_value = "5")] + pub max_unsafe_interval: u64, + /// Disable using the proposer to sync the builder node #[arg(long, env, default_value = "false")] pub no_boost_sync: bool, @@ -169,6 +173,7 @@ impl Args { self.execution_mode, probes, self.health_check_interval, + self.max_unsafe_interval, ); // Spawn the debug server diff --git a/src/client/rpc.rs b/src/client/rpc.rs index c99a3319..4ca8106b 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -2,11 +2,13 @@ use crate::client::auth::AuthLayer; use crate::server::{ EngineApiClient, NewPayload, OpExecutionPayloadEnvelope, PayloadSource, Version, }; -use alloy_primitives::{B256, Bytes, U256}; + +use alloy_primitives::{B256, Bytes}; use alloy_rpc_types_engine::{ ExecutionPayload, ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, JwtError, JwtSecret, PayloadId, PayloadStatus, }; +use alloy_rpc_types_eth::{Block, BlockNumberOrTag}; use clap::{Parser, arg}; use http::Uri; use jsonrpsee::http_client::transport::HttpBackend; @@ -323,9 +325,16 @@ impl RpcClient { } } - pub async fn block_number(&self) -> ClientResult { - let block_number = self.auth_client.block_number().await.set_code()?; - Ok(block_number) + pub async fn get_block_by_number( + &self, + number: BlockNumberOrTag, + full: bool, + ) -> ClientResult { + Ok(self + .auth_client + .get_block_by_number(number, full) + .await + .set_code()?) } } diff --git a/src/health.rs b/src/health.rs index 99b86e10..f4c638ea 100644 --- a/src/health.rs +++ b/src/health.rs @@ -1,49 +1,54 @@ -use std::{sync::Arc, time::Duration}; +use std::{ + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use alloy_rpc_types_eth::BlockNumberOrTag; use tokio::{ task::JoinHandle, time::{Instant, sleep_until}, }; -use tracing::{error, info}; +use tracing::warn; use crate::{Health, Probes, RpcClient}; pub struct HealthHandle { pub probes: Arc, pub builder_client: Arc, - pub l2_client: Arc, pub health_check_interval: u64, + pub max_unsafe_interval: u64, } impl HealthHandle { + /// Periodically checks that the latest unsafe block is not older than the max_unsafe_interval by a specified threshold. pub fn spawn(self) -> JoinHandle<()> { tokio::spawn(async move { loop { - let (l2_block_height, builder_block_height) = tokio::join!( - self.l2_client.block_number(), - self.builder_client.block_number() - ); - match l2_block_height { + let latest_unsafe = match self + .builder_client + .get_block_by_number(BlockNumberOrTag::Latest, false) + .await + { + Ok(block) => block, Err(e) => { - error!(target: "rollup_boost::health", "Failed to get block height from l2 client: {}", e); - self.probes.set_health(Health::ServiceUnavailable); + warn!(target: "rollup_boost::health", "Failed to get unsafe block from builder client: {} - updating health status", e); + self.probes.set_health(Health::PartialContent); + continue; } - Ok(l2_block_height) => match builder_block_height { - Err(e) => { - error!(target: "rollup_boost::health", "Failed to get block height from builder client: {}", e); - self.probes.set_health(Health::PartialContent); - } - Ok(builder_block_height) => { - if builder_block_height != l2_block_height { - error!(target: "rollup_boost::health", "Builder and L2 client block heights do not match: builder: {}, l2: {}", builder_block_height, l2_block_height); - self.probes.set_health(Health::PartialContent); - } else { - info!(target: "rollup_boost::health", %builder_block_height, "Health Status Check Passed - Builder and L2 client block heights match"); - self.probes.set_health(Health::Healthy); - } - } - }, + }; + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + + if now - latest_unsafe.header.timestamp > self.max_unsafe_interval { + warn!(target: "rollup_boost::health", "Unsafe block timestamp is too old ({} seconds - updating health status)", now - latest_unsafe.header.timestamp); + self.probes.set_health(Health::PartialContent); + } else { + self.probes.set_health(Health::Healthy); } + sleep_until(Instant::now() + Duration::from_secs(self.health_check_interval)).await; } }) @@ -54,7 +59,9 @@ impl HealthHandle { mod tests { use std::net::SocketAddr; - use alloy_primitives::U256; + use alloy_consensus::Header; + use alloy_rpc_types_eth::{Block, Header as EthHeader, Transaction}; + use http::Uri; use http_body_util::BodyExt; use hyper::service::service_fn; @@ -78,7 +85,10 @@ mod tests { } impl MockHttpServer { - async fn serve(f: fn(hyper::Request) -> S) -> eyre::Result + async fn serve( + f: fn(hyper::Request, timestamp: u64) -> S, + timestamp: u64, + ) -> eyre::Result where S: Future, hyper::Error>> + Send @@ -96,7 +106,10 @@ mod tests { let io = TokioIo::new(stream); tokio::spawn(async move { if let Err(err) = hyper::server::conn::http1::Builder::new() - .serve_connection(io, service_fn(move |req| f(req))) + .serve_connection( + io, + service_fn(move |req| f(req, timestamp)), + ) .await { eprintln!("Error serving connection: {}", err); @@ -116,8 +129,9 @@ mod tests { } } - async fn handler_0( + async fn handler( req: hyper::Request, + block_timstamp: u64, ) -> Result, hyper::Error> { let body_bytes = match req.into_body().collect().await { Ok(buf) => buf.to_bytes(), @@ -145,58 +159,21 @@ mod tests { let method = request_body["method"].as_str().unwrap_or_default(); - let response = match method { - "eth_blockNumber" => json!({ - "jsonrpc": "2.0", - "result": format!("{}", U256::ZERO), - "id": request_body["id"] - }), - _ => { - let error_response = json!({ - "jsonrpc": "2.0", - "error": { "code": -32601, "message": "Method not found" }, - "id": request_body["id"] - }); - return Ok(hyper::Response::new(error_response.to_string())); - } - }; - - Ok(hyper::Response::new(response.to_string())) - } - - async fn handler_1( - req: hyper::Request, - ) -> Result, hyper::Error> { - let body_bytes = match req.into_body().collect().await { - Ok(buf) => buf.to_bytes(), - Err(_) => { - let error_response = json!({ - "jsonrpc": "2.0", - "error": { "code": -32700, "message": "Failed to read request body" }, - "id": null - }); - return Ok(hyper::Response::new(error_response.to_string())); - } - }; - - let request_body: serde_json::Value = match serde_json::from_slice(&body_bytes) { - Ok(json) => json, - Err(_) => { - let error_response = json!({ - "jsonrpc": "2.0", - "error": { "code": -32700, "message": "Invalid JSON format" }, - "id": null - }); - return Ok(hyper::Response::new(error_response.to_string())); - } + let mock_block = Block:: { + header: EthHeader { + inner: Header { + timestamp: block_timstamp, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() }; - let method = request_body["method"].as_str().unwrap_or_default(); - let response = match method { - "eth_blockNumber" => json!({ + "eth_getBlockByNumber" => json!({ "jsonrpc": "2.0", - "result": format!("{}", U256::from(1)), + "result": mock_block, "id": request_body["id"] }), _ => { @@ -216,26 +193,24 @@ mod tests { async fn test_health_check_healthy() -> eyre::Result<()> { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; let probes = Arc::new(Probes::default()); - let builder = MockHttpServer::serve(handler_0).await.unwrap(); - let l2 = MockHttpServer::serve(handler_0).await.unwrap(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + let builder = MockHttpServer::serve(handler, now).await.unwrap(); let builder_client = Arc::new(RpcClient::new( format!("http://{}", builder.addr).parse::()?, JwtSecret::random(), 100, PayloadSource::Builder, )?); - let l2_client = Arc::new(RpcClient::new( - format!("http://{}", l2.addr).parse::()?, - JwtSecret::random(), - 100, - PayloadSource::L2, - )?); + let health_handle = HealthHandle { probes: probes.clone(), builder_client: builder_client.clone(), - l2_client: l2_client.clone(), health_check_interval: 60, + max_unsafe_interval: 5, }; let _ = health_handle.spawn(); @@ -245,11 +220,14 @@ mod tests { } #[tokio::test] - async fn test_health_check_partial_content() -> eyre::Result<()> { + async fn test_health_check_exceeds_max_unsafe_interval() -> eyre::Result<()> { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; let probes = Arc::new(Probes::default()); - let builder = MockHttpServer::serve(handler_0).await.unwrap(); - let l2 = MockHttpServer::serve(handler_1).await.unwrap(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + let builder = MockHttpServer::serve(handler, now - 10).await.unwrap(); let builder_client = Arc::new(RpcClient::new( format!("http://{}", builder.addr).parse::()?, @@ -257,17 +235,36 @@ mod tests { 100, PayloadSource::Builder, )?); - let l2_client = Arc::new(RpcClient::new( - format!("http://{}", l2.addr).parse::()?, + + let health_handle = HealthHandle { + probes: probes.clone(), + builder_client: builder_client.clone(), + health_check_interval: 60, + max_unsafe_interval: 5, + }; + + let _ = health_handle.spawn(); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(matches!(probes.health(), Health::PartialContent)); + Ok(()) + } + + #[tokio::test] + async fn test_health_check_service_unavailable() -> eyre::Result<()> { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + let probes = Arc::new(Probes::default()); + let builder_client = Arc::new(RpcClient::new( + "http://127.0.0.1:6000".parse::()?, JwtSecret::random(), 100, - PayloadSource::L2, + PayloadSource::Builder, )?); + let health_handle = HealthHandle { probes: probes.clone(), builder_client: builder_client.clone(), - l2_client: l2_client.clone(), health_check_interval: 60, + max_unsafe_interval: 5, }; let _ = health_handle.spawn(); diff --git a/src/server.rs b/src/server.rs index 8dd87ec7..7ebce144 100644 --- a/src/server.rs +++ b/src/server.rs @@ -2,7 +2,8 @@ use crate::HealthHandle; use crate::client::rpc::RpcClient; use crate::debug_api::DebugServer; use crate::probe::{Health, Probes}; -use alloy_primitives::{B256, Bytes, U256}; +use alloy_primitives::{B256, Bytes}; +use alloy_rpc_types_eth::{Block, BlockNumberOrTag}; use metrics::counter; use moka::sync::Cache; use opentelemetry::trace::SpanKind; @@ -142,13 +143,14 @@ impl RollupBoostServer { initial_execution_mode: ExecutionMode, probes: Arc, health_check_interval: u64, + max_unsafe_interval: u64, ) -> Self { // Spawns a helth check service in the background to continuously check the health of the L2 and builder clients let health_handle = HealthHandle { probes: probes.clone(), builder_client: Arc::new(builder_client.clone()), - l2_client: Arc::new(l2_client.clone()), health_check_interval, + max_unsafe_interval, } .spawn(); @@ -257,8 +259,8 @@ pub trait EngineApi { execution_requests: Vec, ) -> RpcResult; - #[method(name = "eth_blockNumber")] - async fn block_number(&self) -> RpcResult; + #[method(name = "eth_getBlockByNumber")] + async fn get_block_by_number(&self, number: BlockNumberOrTag, full: bool) -> RpcResult; } #[async_trait] @@ -430,8 +432,8 @@ impl EngineApiServer for RollupBoostServer { .await } - async fn block_number(&self) -> RpcResult { - Ok(self.l2_client.block_number().await?) + async fn get_block_by_number(&self, number: BlockNumberOrTag, full: bool) -> RpcResult { + Ok(self.l2_client.get_block_by_number(number, full).await?) } } @@ -782,7 +784,8 @@ mod tests { boost_sync, ExecutionMode::Enabled, probes, - 10, + 60, + 5, ); let module: RpcModule<()> = rollup_boost.try_into().unwrap(); From 1cd09aafb051341e3015799e76cd0d2740fa8746 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 25 Apr 2025 15:40:59 -0700 Subject: [PATCH 36/39] chore: update doc comments --- src/cli.rs | 4 ++-- src/health.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index d0d58d3f..d6d9d349 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -27,11 +27,11 @@ pub struct Args { #[clap(flatten)] pub l2_client: L2ClientArgs, - /// Duration in seconds between async health checks on the builder, and the l2 client + /// Duration in seconds between async health checks on the builder #[arg(long, env, default_value = "60")] pub health_check_interval: u64, - /// Max duration in seconds between the unsafe head block and the current time to be considered healthy + /// Max duration in seconds between the unsafe head block of the builder and the current time #[arg(long, env, default_value = "5")] pub max_unsafe_interval: u64, diff --git a/src/health.rs b/src/health.rs index f4c638ea..a68e9aaf 100644 --- a/src/health.rs +++ b/src/health.rs @@ -20,7 +20,8 @@ pub struct HealthHandle { } impl HealthHandle { - /// Periodically checks that the latest unsafe block is not older than the max_unsafe_interval by a specified threshold. + /// Periodically checks that the latest unsafe block timestamp is not older than the + /// the current time minus the max_unsafe_interval. pub fn spawn(self) -> JoinHandle<()> { tokio::spawn(async move { loop { From 0db4e36b0a6e269913513776f118874ef67760d3 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 25 Apr 2025 15:46:44 -0700 Subject: [PATCH 37/39] fix: loop --- src/health.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/health.rs b/src/health.rs index a68e9aaf..af12d107 100644 --- a/src/health.rs +++ b/src/health.rs @@ -20,7 +20,7 @@ pub struct HealthHandle { } impl HealthHandle { - /// Periodically checks that the latest unsafe block timestamp is not older than the + /// Periodically checks that the latest unsafe block timestamp is not older than the /// the current time minus the max_unsafe_interval. pub fn spawn(self) -> JoinHandle<()> { tokio::spawn(async move { @@ -34,6 +34,10 @@ impl HealthHandle { Err(e) => { warn!(target: "rollup_boost::health", "Failed to get unsafe block from builder client: {} - updating health status", e); self.probes.set_health(Health::PartialContent); + sleep_until( + Instant::now() + Duration::from_secs(self.health_check_interval), + ) + .await; continue; } }; From 7cc6c088a2b40ecc2c42d05db30a0622d5f1bd98 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 25 Apr 2025 17:17:26 -0700 Subject: [PATCH 38/39] chore: update comments --- src/server.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index 7ebce144..22aba1b1 100644 --- a/src/server.rs +++ b/src/server.rs @@ -145,7 +145,6 @@ impl RollupBoostServer { health_check_interval: u64, max_unsafe_interval: u64, ) -> Self { - // Spawns a helth check service in the background to continuously check the health of the L2 and builder clients let health_handle = HealthHandle { probes: probes.clone(), builder_client: Arc::new(builder_client.clone()), From 505ad2a6f8e622611f8055de59c0eef158ff0e31 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 25 Apr 2025 17:23:49 -0700 Subject: [PATCH 39/39] merge main --- tests/common/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 5fd97c96..e4489610 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -37,7 +37,6 @@ use testcontainers::{ContainerAsync, ImageExt}; use time::{OffsetDateTime, format_description}; use tokio::io::AsyncWriteExt as _; use tower_http::sensitive_headers::SetSensitiveRequestHeaders; -use tracing::info; /// Default JWT token for testing purposes pub const JWT_SECRET: &str = "688f5d737bad920bdfb2fc2f488d6b6209eebda1dae949a8de91398d932c517a"; @@ -58,11 +57,9 @@ impl LogConsumer for LoggingConsumer { async move { match record { testcontainers::core::logs::LogFrame::StdOut(bytes) => { - info!(target = self.target, "{}", String::from_utf8_lossy(bytes)); self.log_file.lock().await.write_all(bytes).await.unwrap(); } testcontainers::core::logs::LogFrame::StdErr(bytes) => { - info!(target = self.target, "{}", String::from_utf8_lossy(bytes)); self.log_file.lock().await.write_all(bytes).await.unwrap(); } }