diff --git a/api/signer-api.yml b/api/signer-api.yml index 757b9d6c..888b64fb 100644 --- a/api/signer-api.yml +++ b/api/signer-api.yml @@ -5,6 +5,7 @@ info: description: API that allows commit modules to request generic signatures from validators tags: - name: Signer + - name: Management paths: /signer/v1/get_pubkeys: get: @@ -254,6 +255,20 @@ paths: type: string example: "Internal error" + /status: + get: + summary: Get the status of the Signer API module + tags: + - Management + responses: + "200": + description: Success + content: + text/plain: + schema: + type: string + example: "OK" + components: securitySchemes: BearerAuth: diff --git a/bin/src/lib.rs b/bin/src/lib.rs index c280d2a1..55961348 100644 --- a/bin/src/lib.rs +++ b/bin/src/lib.rs @@ -20,7 +20,7 @@ pub mod prelude { pub use cb_metrics::provider::MetricsProvider; pub use cb_pbs::{ get_header, get_status, register_validator, submit_block, BuilderApi, BuilderApiState, - DefaultBuilderApi, PbsService, PbsState, + DefaultBuilderApi, PbsService, PbsState, PbsStateGuard, }; // The TreeHash derive macro requires tree_hash as import pub mod tree_hash { diff --git a/crates/common/src/commit/constants.rs b/crates/common/src/commit/constants.rs index 3335833a..7c9f948c 100644 --- a/crates/common/src/commit/constants.rs +++ b/crates/common/src/commit/constants.rs @@ -2,3 +2,4 @@ pub const GET_PUBKEYS_PATH: &str = "/signer/v1/get_pubkeys"; pub const REQUEST_SIGNATURE_PATH: &str = "/signer/v1/request_signature"; pub const GENERATE_PROXY_KEY_PATH: &str = "/signer/v1/generate_proxy_key"; pub const STATUS_PATH: &str = "/status"; +pub const RELOAD_PATH: &str = "/reload"; diff --git a/crates/common/src/pbs/constants.rs b/crates/common/src/pbs/constants.rs index 3da8d96d..15fb305e 100644 --- a/crates/common/src/pbs/constants.rs +++ b/crates/common/src/pbs/constants.rs @@ -6,6 +6,7 @@ pub const GET_HEADER_PATH: &str = "/header/{slot}/{parent_hash}/{pubkey}"; pub const GET_STATUS_PATH: &str = "/status"; pub const REGISTER_VALIDATOR_PATH: &str = "/validators"; pub const SUBMIT_BLOCK_PATH: &str = "/blinded_blocks"; +pub const RELOAD_PATH: &str = "/reload"; // https://ethereum.github.io/builder-specs/#/Builder diff --git a/crates/common/src/pbs/event.rs b/crates/common/src/pbs/event.rs index d4049781..015de714 100644 --- a/crates/common/src/pbs/event.rs +++ b/crates/common/src/pbs/event.rs @@ -37,6 +37,8 @@ pub enum BuilderEvent { }, RegisterValidatorRequest(Vec), RegisterValidatorResponse, + ReloadEvent, + ReloadResponse, } #[derive(Debug, Clone)] diff --git a/crates/pbs/src/api.rs b/crates/pbs/src/api.rs index 217e438a..8a76a423 100644 --- a/crates/pbs/src/api.rs +++ b/crates/pbs/src/api.rs @@ -7,13 +7,13 @@ use cb_common::pbs::{ use crate::{ mev_boost, - state::{BuilderApiState, PbsState}, + state::{BuilderApiState, PbsState, PbsStateGuard}, }; #[async_trait] pub trait BuilderApi: 'static { /// Use to extend the BuilderApi - fn extra_routes() -> Option>> { + fn extra_routes() -> Option>> { None } @@ -48,6 +48,10 @@ pub trait BuilderApi: 'static { ) -> eyre::Result<()> { mev_boost::register_validator(registrations, req_headers, state).await } + + async fn reload(state: PbsState) -> eyre::Result> { + mev_boost::reload(state).await + } } pub struct DefaultBuilderApi; diff --git a/crates/pbs/src/constants.rs b/crates/pbs/src/constants.rs index 54156d1b..8dc3bd86 100644 --- a/crates/pbs/src/constants.rs +++ b/crates/pbs/src/constants.rs @@ -2,6 +2,7 @@ pub const STATUS_ENDPOINT_TAG: &str = "status"; pub const REGISTER_VALIDATOR_ENDPOINT_TAG: &str = "register_validator"; pub const SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG: &str = "submit_blinded_block"; pub const GET_HEADER_ENDPOINT_TAG: &str = "get_header"; +pub const RELOAD_ENDPOINT_TAG: &str = "reload"; /// For metrics recorded when a request times out pub const TIMEOUT_ERROR_CODE: u16 = 555; diff --git a/crates/pbs/src/error.rs b/crates/pbs/src/error.rs index 74fc7004..590c03d4 100644 --- a/crates/pbs/src/error.rs +++ b/crates/pbs/src/error.rs @@ -5,6 +5,7 @@ use axum::{http::StatusCode, response::IntoResponse}; pub enum PbsClientError { NoResponse, NoPayload, + Internal, } impl PbsClientError { @@ -12,15 +13,17 @@ impl PbsClientError { match self { PbsClientError::NoResponse => StatusCode::BAD_GATEWAY, PbsClientError::NoPayload => StatusCode::BAD_GATEWAY, + PbsClientError::Internal => StatusCode::INTERNAL_SERVER_ERROR, } } } impl IntoResponse for PbsClientError { fn into_response(self) -> axum::response::Response { - let msg = match self { - PbsClientError::NoResponse => "no response from relays", - PbsClientError::NoPayload => "no payload from relays", + let msg = match &self { + PbsClientError::NoResponse => "no response from relays".to_string(), + PbsClientError::NoPayload => "no payload from relays".to_string(), + PbsClientError::Internal => "internal server error".to_string(), }; (self.status_code(), msg).into_response() diff --git a/crates/pbs/src/lib.rs b/crates/pbs/src/lib.rs index efe76d4e..8b4afdcf 100644 --- a/crates/pbs/src/lib.rs +++ b/crates/pbs/src/lib.rs @@ -12,4 +12,4 @@ pub use api::*; pub use constants::*; pub use mev_boost::*; pub use service::PbsService; -pub use state::{BuilderApiState, PbsState}; +pub use state::{BuilderApiState, PbsState, PbsStateGuard}; diff --git a/crates/pbs/src/mev_boost/mod.rs b/crates/pbs/src/mev_boost/mod.rs index e80ee2b8..a41b79db 100644 --- a/crates/pbs/src/mev_boost/mod.rs +++ b/crates/pbs/src/mev_boost/mod.rs @@ -1,9 +1,11 @@ mod get_header; mod register_validator; +mod reload; mod status; mod submit_block; pub use get_header::get_header; pub use register_validator::register_validator; +pub use reload::reload; pub use status::get_status; pub use submit_block::submit_block; diff --git a/crates/pbs/src/mev_boost/reload.rs b/crates/pbs/src/mev_boost/reload.rs new file mode 100644 index 00000000..0a0555b6 --- /dev/null +++ b/crates/pbs/src/mev_boost/reload.rs @@ -0,0 +1,27 @@ +use cb_common::config::load_pbs_config; +use tracing::warn; + +use crate::{BuilderApiState, PbsState}; + +/// Reload the PBS state with the latest configuration in the config file +/// Returns 200 if successful or 500 if failed +pub async fn reload(state: PbsState) -> eyre::Result> { + let pbs_config = load_pbs_config().await?; + let new_state = PbsState::new(pbs_config).with_data(state.data); + + if state.config.pbs_config.host != new_state.config.pbs_config.host { + warn!( + "Host change for PBS module require a full restart. Old: {}, New: {}", + state.config.pbs_config.host, new_state.config.pbs_config.host + ); + } + + if state.config.pbs_config.port != new_state.config.pbs_config.port { + warn!( + "Port change for PBS module require a full restart. Old: {}, New: {}", + state.config.pbs_config.port, new_state.config.pbs_config.port + ); + } + + Ok(new_state) +} diff --git a/crates/pbs/src/routes/get_header.rs b/crates/pbs/src/routes/get_header.rs index 5d0b3bd6..919bad11 100644 --- a/crates/pbs/src/routes/get_header.rs +++ b/crates/pbs/src/routes/get_header.rs @@ -17,15 +17,17 @@ use crate::{ constants::GET_HEADER_ENDPOINT_TAG, error::PbsClientError, metrics::BEACON_NODE_STATUS, - state::{BuilderApiState, PbsState}, + state::{BuilderApiState, PbsStateGuard}, }; #[tracing::instrument(skip_all, name = "get_header", fields(req_id = %Uuid::new_v4(), slot = params.slot))] pub async fn handle_get_header>( - State(state): State>, + State(state): State>, req_headers: HeaderMap, Path(params): Path, ) -> Result { + let state = state.read().clone(); + state.publish_event(BuilderEvent::GetHeaderRequest(params)); let ua = get_user_agent(&req_headers); diff --git a/crates/pbs/src/routes/mod.rs b/crates/pbs/src/routes/mod.rs index 265b7da0..33a07500 100644 --- a/crates/pbs/src/routes/mod.rs +++ b/crates/pbs/src/routes/mod.rs @@ -1,5 +1,6 @@ mod get_header; mod register_validator; +mod reload; mod router; mod status; mod submit_block; diff --git a/crates/pbs/src/routes/register_validator.rs b/crates/pbs/src/routes/register_validator.rs index fd73837b..4566016f 100644 --- a/crates/pbs/src/routes/register_validator.rs +++ b/crates/pbs/src/routes/register_validator.rs @@ -16,15 +16,17 @@ use crate::{ constants::REGISTER_VALIDATOR_ENDPOINT_TAG, error::PbsClientError, metrics::BEACON_NODE_STATUS, - state::{BuilderApiState, PbsState}, + state::{BuilderApiState, PbsStateGuard}, }; #[tracing::instrument(skip_all, name = "register_validators", fields(req_id = %Uuid::new_v4()))] pub async fn handle_register_validator>( - State(state): State>, + State(state): State>, req_headers: HeaderMap, Json(registrations): Json>, ) -> Result { + let state = state.read().clone(); + trace!(?registrations); state.publish_event(BuilderEvent::RegisterValidatorRequest(registrations.clone())); diff --git a/crates/pbs/src/routes/reload.rs b/crates/pbs/src/routes/reload.rs new file mode 100644 index 00000000..9b984d3f --- /dev/null +++ b/crates/pbs/src/routes/reload.rs @@ -0,0 +1,47 @@ +use axum::{extract::State, http::HeaderMap, response::IntoResponse}; +use cb_common::{pbs::BuilderEvent, utils::get_user_agent}; +use reqwest::StatusCode; +use tracing::{error, info}; +use uuid::Uuid; + +use crate::{ + error::PbsClientError, + metrics::BEACON_NODE_STATUS, + state::{BuilderApiState, PbsStateGuard}, + BuilderApi, RELOAD_ENDPOINT_TAG, +}; + +#[tracing::instrument(skip_all, name = "reload", fields(req_id = %Uuid::new_v4()))] +pub async fn handle_reload>( + req_headers: HeaderMap, + State(state): State>, +) -> Result { + let prev_state = state.read().clone(); + + prev_state.publish_event(BuilderEvent::ReloadEvent); + + let ua = get_user_agent(&req_headers); + + info!(ua, relay_check = prev_state.config.pbs_config.relay_check); + + match A::reload(prev_state.clone()).await { + Ok(new_state) => { + prev_state.publish_event(BuilderEvent::ReloadResponse); + info!("config reload successful"); + + *state.write() = new_state; + + BEACON_NODE_STATUS.with_label_values(&["200", RELOAD_ENDPOINT_TAG]).inc(); + Ok((StatusCode::OK, "OK")) + } + Err(err) => { + error!(%err, "config reload failed"); + + let err = PbsClientError::Internal; + BEACON_NODE_STATUS + .with_label_values(&[err.status_code().as_str(), RELOAD_ENDPOINT_TAG]) + .inc(); + Err(err) + } + } +} diff --git a/crates/pbs/src/routes/router.rs b/crates/pbs/src/routes/router.rs index f6ae4202..54659c5c 100644 --- a/crates/pbs/src/routes/router.rs +++ b/crates/pbs/src/routes/router.rs @@ -3,23 +3,28 @@ use axum::{ Router, }; use cb_common::pbs::{ - BUILDER_API_PATH, GET_HEADER_PATH, GET_STATUS_PATH, REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH, + BUILDER_API_PATH, GET_HEADER_PATH, GET_STATUS_PATH, REGISTER_VALIDATOR_PATH, RELOAD_PATH, + SUBMIT_BLOCK_PATH, }; -use super::{handle_get_header, handle_get_status, handle_register_validator, handle_submit_block}; +use super::{ + handle_get_header, handle_get_status, handle_register_validator, handle_submit_block, + reload::handle_reload, +}; use crate::{ api::BuilderApi, - state::{BuilderApiState, PbsState}, + state::{BuilderApiState, PbsStateGuard}, }; -pub fn create_app_router>(state: PbsState) -> Router { +pub fn create_app_router>(state: PbsStateGuard) -> Router { let builder_routes = Router::new() .route(GET_HEADER_PATH, get(handle_get_header::)) .route(GET_STATUS_PATH, get(handle_get_status::)) .route(REGISTER_VALIDATOR_PATH, post(handle_register_validator::)) .route(SUBMIT_BLOCK_PATH, post(handle_submit_block::)); + let reload_router = Router::new().route(RELOAD_PATH, post(handle_reload::)); - let builder_api = Router::new().nest(BUILDER_API_PATH, builder_routes); + let builder_api = Router::new().nest(BUILDER_API_PATH, builder_routes).merge(reload_router); let app = if let Some(extra_routes) = A::extra_routes() { builder_api.merge(extra_routes) diff --git a/crates/pbs/src/routes/status.rs b/crates/pbs/src/routes/status.rs index 2a33e4a8..b4262d1f 100644 --- a/crates/pbs/src/routes/status.rs +++ b/crates/pbs/src/routes/status.rs @@ -9,14 +9,16 @@ use crate::{ constants::STATUS_ENDPOINT_TAG, error::PbsClientError, metrics::BEACON_NODE_STATUS, - state::{BuilderApiState, PbsState}, + state::{BuilderApiState, PbsStateGuard}, }; #[tracing::instrument(skip_all, name = "status", fields(req_id = %Uuid::new_v4()))] pub async fn handle_get_status>( req_headers: HeaderMap, - State(state): State>, + State(state): State>, ) -> Result { + let state = state.read().clone(); + state.publish_event(BuilderEvent::GetStatusEvent); let ua = get_user_agent(&req_headers); diff --git a/crates/pbs/src/routes/submit_block.rs b/crates/pbs/src/routes/submit_block.rs index 5ae64c5a..c9d206a1 100644 --- a/crates/pbs/src/routes/submit_block.rs +++ b/crates/pbs/src/routes/submit_block.rs @@ -12,15 +12,17 @@ use crate::{ constants::SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, error::PbsClientError, metrics::BEACON_NODE_STATUS, - state::{BuilderApiState, PbsState}, + state::{BuilderApiState, PbsStateGuard}, }; #[tracing::instrument(skip_all, name = "submit_blinded_block", fields(req_id = %Uuid::new_v4(), slot = signed_blinded_block.message.slot))] pub async fn handle_submit_block>( - State(state): State>, + State(state): State>, req_headers: HeaderMap, Json(signed_blinded_block): Json, ) -> Result { + let state = state.read().clone(); + trace!(?signed_blinded_block); state.publish_event(BuilderEvent::SubmitBlockRequest(Box::new(signed_blinded_block.clone()))); diff --git a/crates/pbs/src/service.rs b/crates/pbs/src/service.rs index 8cc5d5bb..6b6fd677 100644 --- a/crates/pbs/src/service.rs +++ b/crates/pbs/src/service.rs @@ -7,6 +7,7 @@ use cb_common::{ }; use cb_metrics::provider::MetricsProvider; use eyre::{bail, Context, Result}; +use parking_lot::RwLock; use prometheus::core::Collector; use tokio::net::TcpListener; use tracing::info; @@ -28,7 +29,7 @@ impl PbsService { state.config.event_publisher.as_ref().map(|e| e.n_subscribers()).unwrap_or_default(); info!(version = COMMIT_BOOST_VERSION, commit = COMMIT_BOOST_COMMIT, ?addr, events_subs, chain =? state.config.chain, "starting PBS service"); - let app = create_app_router::(state); + let app = create_app_router::(RwLock::new(state).into()); let listener = TcpListener::bind(addr).await?; let task = diff --git a/crates/pbs/src/state.rs b/crates/pbs/src/state.rs index 6b3f15c9..d2bb2eb9 100644 --- a/crates/pbs/src/state.rs +++ b/crates/pbs/src/state.rs @@ -1,12 +1,17 @@ +use std::sync::Arc; + use alloy::rpc::types::beacon::BlsPublicKey; use cb_common::{ config::{PbsConfig, PbsModuleConfig}, pbs::{BuilderEvent, RelayClient}, }; +use parking_lot::RwLock; pub trait BuilderApiState: Clone + Sync + Send + 'static {} impl BuilderApiState for () {} +pub type PbsStateGuard = Arc>>; + /// Config for the Pbs module. It can be extended by adding extra data to the /// state for modules that need it // TODO: consider remove state from the PBS module altogether diff --git a/crates/signer/src/service.rs b/crates/signer/src/service.rs index c32e673c..f8544137 100644 --- a/crates/signer/src/service.rs +++ b/crates/signer/src/service.rs @@ -13,7 +13,8 @@ use bimap::BiHashMap; use cb_common::{ commit::{ constants::{ - GENERATE_PROXY_KEY_PATH, GET_PUBKEYS_PATH, REQUEST_SIGNATURE_PATH, STATUS_PATH, + GENERATE_PROXY_KEY_PATH, GET_PUBKEYS_PATH, RELOAD_PATH, REQUEST_SIGNATURE_PATH, + STATUS_PATH, }, request::{ EncryptionScheme, GenerateProxyRequest, GetPubkeysResponse, SignConsensusRequest, @@ -25,7 +26,7 @@ use cb_common::{ types::{Chain, Jwt, ModuleId}, }; use cb_metrics::provider::MetricsProvider; -use eyre::{Context, Result}; +use eyre::Context; use headers::{authorization::Bearer, Authorization}; use tokio::{net::TcpListener, sync::RwLock}; use tracing::{debug, error, info, warn}; @@ -56,18 +57,9 @@ impl SigningService { return Ok(()); } - let proxy_store = if let Some(store) = config.store { - Some(store.init_from_env()?) - } else { - warn!("Proxy store not configured. Proxies keys and delegations will not be persisted"); - None - }; + let manager = start_manager(&config) + .map_err(|err| eyre::eyre!("failed to start signing manager {err}"))?; - let mut manager = SigningManager::new(config.chain, proxy_store)?; - - for signer in config.loader.load_keys()? { - manager.add_consensus_signer(signer); - } let module_ids: Vec = config.jwts.left_values().cloned().map(Into::into).collect(); let loaded_consensus = manager.consensus_pubkeys().len(); @@ -83,8 +75,9 @@ impl SigningService { .route(REQUEST_SIGNATURE_PATH, post(handle_request_signature)) .route(GET_PUBKEYS_PATH, get(handle_get_pubkeys)) .route(GENERATE_PROXY_KEY_PATH, post(handle_generate_proxy)) - .with_state(state.clone()) .route_layer(middleware::from_fn_with_state(state.clone(), jwt_auth)) + .route(RELOAD_PATH, post(handle_reload)) + .with_state(state.clone()) .route_layer(middleware::from_fn(log_request)); let status_router = axum::Router::new().route(STATUS_PATH, get(handle_status)); @@ -96,7 +89,7 @@ impl SigningService { .wrap_err("signer server exited") } - fn init_metrics(network: Chain) -> Result<()> { + fn init_metrics(network: Chain) -> eyre::Result<()> { MetricsProvider::load_and_run(network, SIGNER_METRICS_REGISTRY.clone()) } } @@ -220,3 +213,48 @@ async fn handle_generate_proxy( Ok(response) } + +async fn handle_reload( + State(state): State, +) -> Result { + let req_id = Uuid::new_v4(); + + debug!(event = "reload", ?req_id, "New request"); + + let config = match StartSignerConfig::load_from_env() { + Ok(config) => config, + Err(err) => { + error!(event = "reload", ?req_id, error = ?err, "Failed to reload config"); + return Err(SignerModuleError::Internal("failed to reload config".to_string())); + } + }; + + let new_manager = match start_manager(&config) { + Ok(manager) => manager, + Err(err) => { + error!(event = "reload", ?req_id, error = ?err, "Failed to reload manager"); + return Err(SignerModuleError::Internal("failed to reload config".to_string())); + } + }; + + *state.manager.write().await = new_manager; + + Ok((StatusCode::OK, "OK")) +} + +fn start_manager(config: &StartSignerConfig) -> eyre::Result { + let proxy_store = if let Some(store) = config.store.clone() { + Some(store.init_from_env()?) + } else { + warn!("Proxy store not configured. Proxies keys and delegations will not be persisted"); + None + }; + + let mut manager = SigningManager::new(config.chain, proxy_store)?; + + for signer in config.loader.clone().load_keys()? { + manager.add_consensus_signer(signer); + } + + Ok(manager) +} diff --git a/docs/docs/get_started/configuration.md b/docs/docs/get_started/configuration.md index 36462642..0f78b4d6 100644 --- a/docs/docs/get_started/configuration.md +++ b/docs/docs/get_started/configuration.md @@ -311,3 +311,19 @@ This approach could also work if you have a multi-beacon-node setup, where some ### Notes - It's up to you to decide which relays will be connected via Commit-Boost (`[[relays]]` section in the `toml` config) and which via Vouch (additional entries in the `relays` field). Remember that any rate-limit will be shared across the two sidecars, if running on the same machine. - You may occasionally see a `timeout` error during registrations, especially if you're running a large number of validators in the same instance. This can resolve itself as registrations will be cleared later in the epoch when relays are less busy processing other registrations. Alternatively you can also adjust the `builderclient.timeout` option in `.vouch.yml`. + +## Hot Reload + +Commit-Boost supports hot-reloading the configuration file. This means that you can modify the `cb-config.toml` file and apply the changes without needing to restart the modules. To do this, you need to send a `POST` request to the `/reload` endpoint on each module you want to reload the configuration. In the case the module is running in a Docker container without the port exposed (like the signer), you can use the following command: + +```bash +docker compose -f cb.docker-compose.yml exec cb_signer curl -X POST http://localhost:20000/reload +``` + +### Notes + +- The hot reload feature is available for PBS modules (both default and custom) and signer module. +- Changes related to listening hosts and ports will not been applied, as it requires the server to be restarted. +- If running in Docker containers, changes in `volumes` will not be applied, as it requires the container to be recreated. Be careful if changing a path to a local file as it may not be accessible from the container. +- Custom PBS modules may override the default behaviour of the hot reload feature to parse extra configuration fields. Check the [examples](https://github.com/Commit-Boost/commit-boost-client/blob/main/examples/status_api/src/main.rs) for more details. +- In case the reload fails (most likely because of some misconfigured option), the server will return a 500 error and the previous configuration will be kept. diff --git a/examples/status_api/src/main.rs b/examples/status_api/src/main.rs index 483beefc..973348bc 100644 --- a/examples/status_api/src/main.rs +++ b/examples/status_api/src/main.rs @@ -65,15 +65,23 @@ impl BuilderApi for MyBuilderApi { get_status(req_headers, state).await } - fn extra_routes() -> Option>> { + async fn reload(state: PbsState) -> Result> { + let (pbs_config, extra_config) = load_pbs_custom_config::().await?; + let mut data = state.data.clone(); + data.inc_amount = extra_config.inc_amount; + + Ok(PbsState::new(pbs_config).with_data(data)) + } + + fn extra_routes() -> Option>> { let mut router = Router::new(); router = router.route("/check", get(handle_check)); Some(router) } } -async fn handle_check(State(state): State>) -> Response { - (StatusCode::OK, format!("Received {count} status requests!", count = state.data.get())) +async fn handle_check(State(state): State>) -> Response { + (StatusCode::OK, format!("Received {count} status requests!", count = state.read().data.get())) .into_response() }