diff --git a/Cargo.toml b/Cargo.toml index bb5d8f23..c5350e75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ dashmap = "5.5.3" blst = "0.3.11" tree_hash = "0.5" tree_hash_derive = "0.5" +eth2_keystore = { git = "https://github.com/sigp/lighthouse", rev = "9e12c21f268c80a3f002ae0ca27477f9f512eb6f" } # docker docker-compose-types = "0.12.0" diff --git a/config.example.toml b/config.example.toml index f662324f..900242e1 100644 --- a/config.example.toml +++ b/config.example.toml @@ -15,7 +15,8 @@ X-MyCustomHeader = "MyCustomValue" [signer] [signer.loader] -key_path = "keys.example.json" +keys_path = "" +secrets_path = "" [metrics] prometheus_config = "./docker/prometheus.yml" diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 8ffdb032..7cfa0a7e 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -2,10 +2,11 @@ use std::{path::Path, vec}; use cb_common::{ config::{ - CommitBoostConfig, SignerLoader, CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV, - METRICS_SERVER_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, SIGNER_LOADER_ENV, SIGNER_LOADER_NAME, - SIGNER_SERVER_ENV, + CommitBoostConfig, CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV, METRICS_SERVER_ENV, + MODULE_ID_ENV, MODULE_JWT_ENV, SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS, + SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, SIGNER_KEYS_ENV, SIGNER_SERVER_ENV, }, + loader::SignerLoader, utils::random_jwt, }; use docker_compose_types::{ @@ -123,27 +124,38 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> eyre::Resu // setup signer service if let Some(signer_config) = cb_config.signer { - // TODO: generalize this, different loaders may not need volumes but eg ports - let signer_volume = match signer_config.loader { - SignerLoader::File { key_path } => { - Volumes::Simple(format!("./{}:{}:ro", key_path, SIGNER_LOADER_NAME)) - } - }; + let mut volumes = vec![config_volume.clone()]; targets.push(PrometheusTargetConfig { targets: vec![format!("cb_signer:{metrics_port}")], labels: PrometheusLabelsConfig { job: "signer".into() }, }); - let signer_envs = IndexMap::from([ + let mut signer_envs = IndexMap::from([ get_env_same(CB_CONFIG_ENV), - get_env_same(SIGNER_LOADER_ENV), get_env_same(JWTS_ENV), get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()), get_env_val(SIGNER_SERVER_ENV, &signer_port.to_string()), ]); - envs.insert(SIGNER_LOADER_ENV.into(), SIGNER_LOADER_NAME.into()); + // TODO: generalize this, different loaders may not need volumes but eg ports + match signer_config.loader { + SignerLoader::File { key_path } => { + volumes.push(Volumes::Simple(format!("./{}:{}:ro", key_path, SIGNER_KEYS))); + let (k, v) = get_env_val(SIGNER_KEYS_ENV, SIGNER_KEYS); + signer_envs.insert(k, v); + } + SignerLoader::ValidatorsDir { keys_path, secrets_path } => { + volumes.push(Volumes::Simple(format!("{}:{}:ro", keys_path, SIGNER_DIR_KEYS))); + let (k, v) = get_env_val(SIGNER_DIR_KEYS_ENV, SIGNER_DIR_KEYS); + signer_envs.insert(k, v); + + volumes + .push(Volumes::Simple(format!("{}:{}:ro", secrets_path, SIGNER_DIR_SECRETS))); + let (k, v) = get_env_val(SIGNER_DIR_SECRETS_ENV, SIGNER_DIR_SECRETS); + signer_envs.insert(k, v); + } + }; // write jwts to env let jwts_json = serde_json::to_string(&jwts).unwrap().clone(); @@ -153,7 +165,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> eyre::Resu container_name: Some("cb_signer".to_owned()), image: Some(signer_config.docker_image), networks: Networks::Simple(vec![METRICS_NETWORK.to_owned(), SIGNER_NETWORK.to_owned()]), - volumes: vec![config_volume.clone(), signer_volume], + volumes, environment: Environment::KvPair(signer_envs), ..Service::default() }; diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index fec0d826..ad734367 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -27,3 +27,4 @@ reqwest.workspace = true thiserror.workspace = true eyre.workspace = true +eth2_keystore.workspace = true diff --git a/crates/common/src/config.rs b/crates/common/src/config.rs index 57f1c6a8..311f9abe 100644 --- a/crates/common/src/config.rs +++ b/crates/common/src/config.rs @@ -2,13 +2,10 @@ use std::{collections::HashMap, sync::Arc}; use alloy_primitives::U256; use eyre::{eyre, ContextCompat}; -use serde::{ - de::{self, DeserializeOwned}, - Deserialize, Deserializer, Serialize, -}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use super::utils::as_eth_str; -use crate::{commit::client::SignerClient, pbs::RelayEntry, signer::Signer, types::Chain}; +use crate::{commit::client::SignerClient, loader::SignerLoader, pbs::RelayEntry, types::Chain}; pub const MODULE_ID_ENV: &str = "CB_MODULE_ID"; pub const MODULE_JWT_ENV: &str = "CB_SIGNER_JWT"; @@ -18,8 +15,12 @@ pub const SIGNER_SERVER_ENV: &str = "SIGNER_SERVER"; pub const CB_CONFIG_ENV: &str = "CB_CONFIG"; pub const CB_CONFIG_NAME: &str = "/cb-config.toml"; -pub const SIGNER_LOADER_ENV: &str = "CB_SIGNER_LOADER_FILE"; -pub const SIGNER_LOADER_NAME: &str = "/keys.json"; +pub const SIGNER_KEYS_ENV: &str = "CB_SIGNER_FILE"; +pub const SIGNER_KEYS: &str = "/keys.json"; +pub const SIGNER_DIR_KEYS_ENV: &str = "SIGNER_LOADER_DIR_KEYS"; +pub const SIGNER_DIR_KEYS: &str = "/keys"; +pub const SIGNER_DIR_SECRETS_ENV: &str = "SIGNER_LOADER_DIR_SECRETS"; +pub const SIGNER_DIR_SECRETS: &str = "/secrets"; pub const JWTS_ENV: &str = "CB_JWTS"; @@ -121,41 +122,8 @@ impl ModuleMetricsConfig { } } -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(untagged)] -pub enum SignerLoader { - /// Plain text, do not use in prod - File { key_path: String }, -} - -impl SignerLoader { - pub fn load_keys(self) -> Vec { - // TODO: add flag to support also native loader - self.load_from_env() - } - - pub fn load_from_env(self) -> Vec { - match self { - SignerLoader::File { .. } => { - let path = std::env::var(SIGNER_LOADER_ENV) - .expect(&format!("{SIGNER_LOADER_ENV} is not set")); - let file = - std::fs::read_to_string(path).expect(&format!("Unable to find keys file")); - - let keys: Vec = serde_json::from_str(&file).unwrap(); - - keys.into_iter().map(|k| Signer::new_from_bytes(&k.secret_key)).collect() - } - } - } -} - -pub struct FileKey { - pub secret_key: [u8; 32], -} - /// Static pbs config from config file -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Default, Deserialize, Serialize)] pub struct StaticPbsConfig { /// Docker image of the module #[serde(default = "default_pbs")] @@ -168,7 +136,7 @@ pub struct StaticPbsConfig { pub with_signer: bool, } -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct PbsConfig { /// Port to receive BuilderAPI calls from CL pub port: u16, @@ -350,43 +318,7 @@ pub fn load_module_config() -> eyre::Result Deserialize<'de> for FileKey { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - let s = - alloy_primitives::hex::decode(s.trim_start_matches("0x")).map_err(de::Error::custom)?; - let bytes: [u8; 32] = s.try_into().map_err(|_| de::Error::custom("wrong lenght"))?; - - Ok(FileKey { secret_key: bytes }) - } -} - // TODO: propagate errors -fn load_env_var_infallible(env: &str) -> String { +pub fn load_env_var_infallible(env: &str) -> String { std::env::var(env).expect(&format!("{env} is not set")) } - -#[cfg(test)] -mod tests { - - use super::FileKey; - - #[test] - fn test_decode() { - let s = [ - 0, 136, 227, 100, 165, 57, 106, 129, 181, 15, 235, 189, 200, 120, 70, 99, 251, 144, - 137, 181, 230, 124, 189, 193, 115, 153, 26, 0, 197, 135, 103, 63, - ]; - - let d = r#"[ - "0088e364a5396a81b50febbdc8784663fb9089b5e67cbdc173991a00c587673f", - "0088e364a5396a81b50febbdc8784663fb9089b5e67cbdc173991a00c587673f" -]"#; - let decoded: Vec = serde_json::from_str(d).unwrap(); - - assert_eq!(decoded[0].secret_key, s) - } -} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index fe646275..8d47c1a4 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -1,6 +1,7 @@ pub mod commit; pub mod config; pub mod constants; +pub mod loader; pub mod pbs; pub mod signature; pub mod signer; diff --git a/crates/common/src/loader.rs b/crates/common/src/loader.rs new file mode 100644 index 00000000..2272645b --- /dev/null +++ b/crates/common/src/loader.rs @@ -0,0 +1,130 @@ +use std::fs; + +use alloy_primitives::hex::FromHex; +use alloy_rpc_types_beacon::BlsPublicKey; +use eth2_keystore::Keystore; +use eyre::eyre; +use serde::{de, Deserialize, Deserializer, Serialize}; + +use crate::{ + config::{ + load_env_var_infallible, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS_ENV, + }, + signer::Signer, +}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(untagged)] +pub enum SignerLoader { + /// Plain text, do not use in prod + File { + key_path: String, + }, + ValidatorsDir { + keys_path: String, + secrets_path: String, + }, +} + +impl SignerLoader { + pub fn load_keys(self) -> Vec { + // TODO: add flag to support also native loader + self.load_from_env() + } + + pub fn load_from_env(self) -> Vec { + match self { + SignerLoader::File { .. } => { + let path = load_env_var_infallible(SIGNER_KEYS_ENV); + let file = + std::fs::read_to_string(path).expect(&format!("Unable to find keys file")); + + let keys: Vec = serde_json::from_str(&file).unwrap(); + + keys.into_iter().map(|k| Signer::new_from_bytes(&k.secret_key)).collect() + } + SignerLoader::ValidatorsDir { .. } => { + // TODO: hacky way to load for now, we should support reading the definitions.yml + // file + let keys_path = load_env_var_infallible(SIGNER_DIR_KEYS_ENV); + let secrets_path = load_env_var_infallible(SIGNER_DIR_SECRETS_ENV); + load_secrets_and_keys(keys_path, secrets_path).expect("failed to load signers") + } + } + } +} + +pub struct FileKey { + pub secret_key: [u8; 32], +} + +impl<'de> Deserialize<'de> for FileKey { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let s = + alloy_primitives::hex::decode(s.trim_start_matches("0x")).map_err(de::Error::custom)?; + let bytes: [u8; 32] = s.try_into().map_err(|_| de::Error::custom("wrong lenght"))?; + + Ok(FileKey { secret_key: bytes }) + } +} + +fn load_secrets_and_keys(keys_path: String, secrets_path: String) -> eyre::Result> { + let entries = fs::read_dir(keys_path.clone())?; + + let mut signers = Vec::new(); + + for entry in entries { + let entry = entry?; + let path = entry.path(); + + // Check if file name is a pubkey + if path.is_dir() { + if let Some(maybe_pubkey) = path.file_name().and_then(|d| d.to_str()) { + if let Ok(pubkey) = BlsPublicKey::from_hex(maybe_pubkey) { + let ks_path = format!("{}/{}/voting-keystore.json", keys_path, maybe_pubkey); + let pw_path = format!("{}/{}", secrets_path, pubkey); + + if let Ok(signer) = load_one(ks_path, pw_path) { + signers.push(signer); + } + } + }; + } + } + + Ok(signers) +} + +fn load_one(ks_path: String, pw_path: String) -> eyre::Result { + let keystore = Keystore::from_json_file(ks_path).map_err(|_| eyre!("failed reading json"))?; + let password = fs::read(pw_path)?; + let key = + keystore.decrypt_keypair(&password).map_err(|_| eyre!("failed decrypting keypair"))?; + Ok(Signer::new_from_bytes(key.sk.serialize().as_bytes())) +} + +#[cfg(test)] +mod tests { + + use super::FileKey; + + #[test] + fn test_decode() { + let s = [ + 0, 136, 227, 100, 165, 57, 106, 129, 181, 15, 235, 189, 200, 120, 70, 99, 251, 144, + 137, 181, 230, 124, 189, 193, 115, 153, 26, 0, 197, 135, 103, 63, + ]; + + let d = r#"[ + "0088e364a5396a81b50febbdc8784663fb9089b5e67cbdc173991a00c587673f", + "0088e364a5396a81b50febbdc8784663fb9089b5e67cbdc173991a00c587673f" +]"#; + let decoded: Vec = serde_json::from_str(d).unwrap(); + + assert_eq!(decoded[0].secret_key, s) + } +} diff --git a/crates/common/src/signer.rs b/crates/common/src/signer.rs index df8dbc15..556201e1 100644 --- a/crates/common/src/signer.rs +++ b/crates/common/src/signer.rs @@ -10,34 +10,34 @@ use crate::{ #[derive(Clone)] pub enum Signer { - Plain(SecretKey), + Local(SecretKey), } impl Signer { pub fn new_random() -> Self { - Signer::Plain(random_secret()) + Signer::Local(random_secret()) } pub fn new_from_bytes(bytes: &[u8]) -> Self { let secret_key = SecretKey::from_bytes(bytes).unwrap(); - Self::Plain(secret_key) + Self::Local(secret_key) } pub fn pubkey(&self) -> BlsPublicKey { match self { - Signer::Plain(secret) => blst_pubkey_to_alloy(&secret.sk_to_pk()), + Signer::Local(secret) => blst_pubkey_to_alloy(&secret.sk_to_pk()), } } pub async fn sign(&self, chain: Chain, object_root: &[u8; 32]) -> BlsSignature { match self { - Signer::Plain(sk) => sign_builder_message(chain, sk, object_root), + Signer::Local(sk) => sign_builder_message(chain, sk, object_root), } } pub async fn sign_msg(&self, chain: Chain, msg: &impl TreeHash) -> BlsSignature { match self { - Signer::Plain(sk) => { + Signer::Local(sk) => { let object_root = msg.tree_hash_root(); sign_builder_message(chain, sk, &object_root.0) } diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 878af064..15bca8ad 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -109,7 +109,7 @@ pub fn initialize_tracing_log() { tracing_subscriber::fmt() .compact() .with_max_level(level) - .with_target(false) + .with_target(true) .with_file(true) .init(); } diff --git a/crates/metrics/src/sdk.rs b/crates/metrics/src/sdk.rs index 4f3cd3f9..5765641f 100644 --- a/crates/metrics/src/sdk.rs +++ b/crates/metrics/src/sdk.rs @@ -10,7 +10,7 @@ use axum::{ use cb_common::config::ModuleMetricsConfig; use prometheus::{Encoder, Registry, TextEncoder}; use tokio::net::TcpListener; -use tracing::{debug, error, info}; +use tracing::{error, info, trace}; pub struct MetricsProvider { config: ModuleMetricsConfig, @@ -51,7 +51,7 @@ impl MetricsProvider { } async fn handle_metrics(State(registry): State) -> Response { - debug!("Handling metrics request"); + trace!("Handling metrics request"); match prepare_metrics(registry) { Ok(response) => response, diff --git a/crates/pbs/src/metrics.rs b/crates/pbs/src/metrics.rs index 22270344..ab4ab676 100644 --- a/crates/pbs/src/metrics.rs +++ b/crates/pbs/src/metrics.rs @@ -15,7 +15,7 @@ lazy_static! { ]) .unwrap(); pub static ref RELAY_RESPONSE_TIME: HistogramVec = - HistogramVec::new(histogram_opts!("relay_response_time_ms", "Relay response times"), &[ + HistogramVec::new(histogram_opts!("relay_response_time", "Relay response times"), &[ "endpoint", "relay_id" ]) .unwrap(); diff --git a/crates/pbs/src/routes/get_header.rs b/crates/pbs/src/routes/get_header.rs index c2d0bfbb..963fcefd 100644 --- a/crates/pbs/src/routes/get_header.rs +++ b/crates/pbs/src/routes/get_header.rs @@ -29,23 +29,23 @@ pub async fn handle_get_header>( let slot_start_ms = timestamp_of_slot_start_millis(params.slot, state.config.chain); let ua = get_user_agent(&req_headers); - info!(method = "get_header", %req_id, ?ua, slot=params.slot, parent_hash=%params.parent_hash, validator_pubkey=%params.pubkey, ms_into_slot=now.saturating_sub(slot_start_ms)); + info!(event = "get_header", %req_id, ?ua, slot=params.slot, parent_hash=%params.parent_hash, validator_pubkey=%params.pubkey, ms_into_slot=now.saturating_sub(slot_start_ms)); match T::get_header(params, req_headers, state.clone()).await { Ok(res) => { state.publish_event(BuilderEvent::GetHeaderResponse(Box::new(res.clone()))); if let Some(max_bid) = res { - info!(method="get_header", %req_id, block_hash =% max_bid.data.message.header.block_hash, "header available for slot"); + info!(event ="get_header", %req_id, block_hash =% max_bid.data.message.header.block_hash, "header available for slot"); Ok((StatusCode::OK, axum::Json(max_bid)).into_response()) } else { // spec: return 204 if request is valid but no bid available - info!(method="get_header", %req_id, "no header available for slot"); + info!(event = "get_header", %req_id, "no header available for slot"); Ok(StatusCode::NO_CONTENT.into_response()) } } Err(err) => { - error!(method = "get_header", %req_id, ?err, "failed relay get_header"); + error!(event = "get_header", %req_id, ?err, "failed relay get_header"); Err(PbsClientError::NoPayload) } }