From 579d924e0ba0a5992f9aa38a8a7332a6d7bf69e5 Mon Sep 17 00:00:00 2001 From: fbrv Date: Thu, 1 Aug 2024 17:19:54 +0100 Subject: [PATCH 01/22] hourly log rotation --- Cargo.toml | 1 + crates/common/Cargo.toml | 1 + crates/common/src/utils.rs | 17 +++++++++++++---- crates/pbs/src/mev_boost/get_header.rs | 12 ++++++------ crates/pbs/src/mev_boost/status.rs | 4 ++-- crates/pbs/src/mev_boost/submit_block.rs | 10 +++++----- 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 908dcf91..08d2256d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ serde_yaml = "0.9.33" # telemetry tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +tracing-appender = "0.2.3" prometheus = "0.13.4" # crypto diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 03b09e8b..000473a7 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -21,6 +21,7 @@ serde_json.workspace = true # telemetry tracing.workspace = true tracing-subscriber.workspace = true +tracing-appender.workspace = true # crypto blst.workspace = true diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 6cb17cfc..815adb34 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -1,4 +1,7 @@ -use std::time::{SystemTime, UNIX_EPOCH}; +use std::{ + fmt::Debug, + time::{SystemTime, UNIX_EPOCH}, +}; use alloy::{ primitives::U256, @@ -7,7 +10,10 @@ use alloy::{ use blst::min_pk::{PublicKey, Signature}; use rand::{distributions::Alphanumeric, Rng}; use reqwest::header::HeaderMap; -use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; +use tracing_appender::rolling::{RollingFileAppender, Rotation}; +use tracing_subscriber::{ + fmt, fmt::writer::MakeWriterExt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, +}; use crate::types::Chain; @@ -113,7 +119,10 @@ pub const fn default_u256() -> U256 { // TODO: more customized logging + logging guard pub fn initialize_tracing_log() { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); - + // Log all events to a rolling log file. + let logfile = tracing_appender::rolling::hourly("/logs", "myapp-logs"); + // Log `INFO` and above to stdout. + let stdout = std::io::stdout.with_max_level(tracing::Level::INFO); let filter = match level_env.parse::() { Ok(f) => f, Err(_) => { @@ -124,7 +133,7 @@ pub fn initialize_tracing_log() { tracing_subscriber::registry() .with(filter) - .with(fmt::layer().with_target(false)) + .with(fmt::layer().with_writer(stdout.and(logfile))) .try_init() .unwrap(); } diff --git a/crates/pbs/src/mev_boost/get_header.rs b/crates/pbs/src/mev_boost/get_header.rs index b6bd23f5..f1b648f8 100644 --- a/crates/pbs/src/mev_boost/get_header.rs +++ b/crates/pbs/src/mev_boost/get_header.rs @@ -45,7 +45,7 @@ pub async fn get_header( "late in slot, skipping relay requests" ); - return Ok(None) + return Ok(None); } let (_, slot_uuid) = state.get_slot_and_uuid(); @@ -166,7 +166,7 @@ async fn send_timed_get_header( .max_by_key(|(start_time, _)| *start_time) { debug!(n_headers, "TG: received headers from relay"); - return Ok(maybe_header) + return Ok(maybe_header); } else { // all requests failed warn!("TG: no headers received"); @@ -174,7 +174,7 @@ async fn send_timed_get_header( return Err(PbsError::RelayResponse { error_msg: "no headers received".to_string(), code: TIMEOUT_ERROR_CODE, - }) + }); } } } @@ -253,7 +253,7 @@ async fn send_one_get_header( response = ?response_bytes, "no header from relay" ); - return Ok((start_request_time, None)) + return Ok((start_request_time, None)); } let get_header_response: GetHeaderReponse = serde_json::from_slice(&response_bytes)?; @@ -291,7 +291,7 @@ fn validate_header( let value = signed_header.message.value(); if block_hash == B256::ZERO { - return Err(ValidationError::EmptyBlockhash) + return Err(ValidationError::EmptyBlockhash); } if parent_hash != signed_header.message.header.parent_hash { @@ -313,7 +313,7 @@ fn validate_header( return Err(ValidationError::PubkeyMismatch { expected: expected_relay_pubkey, got: received_relay_pubkey, - }) + }); } if !skip_sig_verify { diff --git a/crates/pbs/src/mev_boost/status.rs b/crates/pbs/src/mev_boost/status.rs index 463198c7..cd025cbd 100644 --- a/crates/pbs/src/mev_boost/status.rs +++ b/crates/pbs/src/mev_boost/status.rs @@ -63,7 +63,7 @@ async fn send_relay_check(relay: &RelayClient, headers: HeaderMap) -> Result<(), RELAY_STATUS_CODE .with_label_values(&[TIMEOUT_ERROR_CODE_STR, STATUS_ENDPOINT_TAG, &relay.id]) .inc(); - return Err(err.into()) + return Err(err.into()); } }; let request_latency = start_request.elapsed(); @@ -82,7 +82,7 @@ async fn send_relay_check(relay: &RelayClient, headers: HeaderMap) -> Result<(), }; error!(?err, "status failed"); - return Err(err) + return Err(err); }; debug!(?code, latency = ?request_latency, "status passed"); diff --git a/crates/pbs/src/mev_boost/submit_block.rs b/crates/pbs/src/mev_boost/submit_block.rs index 853eccb6..ff30168b 100644 --- a/crates/pbs/src/mev_boost/submit_block.rs +++ b/crates/pbs/src/mev_boost/submit_block.rs @@ -81,7 +81,7 @@ async fn send_submit_block( &relay.id, ]) .inc(); - return Err(err.into()) + return Err(err.into()); } }; let request_latency = start_request.elapsed(); @@ -103,7 +103,7 @@ async fn send_submit_block( // we request payload to all relays, but some may have not received it warn!(?err, "failed to get payload (this might be ok if other relays have it)"); - return Err(err) + return Err(err); }; let block_response: SubmitBlindedBlockResponse = serde_json::from_slice(&response_bytes)?; @@ -118,7 +118,7 @@ async fn send_submit_block( return Err(PbsError::Validation(ValidationError::BlockHashMismatch { expected: signed_blinded_block.block_hash(), got: block_response.block_hash(), - })) + })); } if let Some(blobs) = &block_response.data.blobs_bundle { @@ -132,7 +132,7 @@ async fn send_submit_block( got_blobs: blobs.blobs.len(), got_commitments: blobs.commitments.len(), got_proofs: blobs.proofs.len(), - })) + })); } for (i, comm) in expected_committments.iter().enumerate() { @@ -142,7 +142,7 @@ async fn send_submit_block( expected: format!("{comm}"), got: format!("{}", blobs.commitments[i]), index: i, - })) + })); } } } From c1f6e6fdb82c4c461c1634bde70f136102dfa61f Mon Sep 17 00:00:00 2001 From: fbrv Date: Thu, 1 Aug 2024 17:21:16 +0100 Subject: [PATCH 02/22] prefix --- crates/common/src/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 815adb34..b1cc574e 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -117,10 +117,10 @@ pub const fn default_u256() -> U256 { // LOGGING // TODO: more customized logging + logging guard -pub fn initialize_tracing_log() { +pub fn initialize_tracing_log(file_name_prefix: &str) { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); // Log all events to a rolling log file. - let logfile = tracing_appender::rolling::hourly("/logs", "myapp-logs"); + let logfile = tracing_appender::rolling::hourly(format!("/logs/{file_name_prefix}"), file_name_prefix); // Log `INFO` and above to stdout. let stdout = std::io::stdout.with_max_level(tracing::Level::INFO); let filter = match level_env.parse::() { From b144c262b039aba7691ee26c8dd939d336bb853a Mon Sep 17 00:00:00 2001 From: fbrv Date: Fri, 2 Aug 2024 13:26:15 +0100 Subject: [PATCH 03/22] implemente log rotation --- bin/default_pbs.rs | 3 +-- bin/signer_module.rs | 3 +-- config.example.toml | 4 ++++ crates/common/src/config.rs | 39 ++++++++++++++++++++++++++++++++++ crates/common/src/utils.rs | 28 +++++++++++++++++------- examples/da_commit/src/main.rs | 3 +-- tests/tests/pbs_integration.rs | 1 + 7 files changed, 67 insertions(+), 14 deletions(-) diff --git a/bin/default_pbs.rs b/bin/default_pbs.rs index 0e5bf8a3..48de6db5 100644 --- a/bin/default_pbs.rs +++ b/bin/default_pbs.rs @@ -11,10 +11,9 @@ async fn main() -> Result<()> { std::env::set_var("RUST_BACKTRACE", "1"); } - initialize_tracing_log(); - // TODO: handle errors let pbs_config = load_pbs_config().expect("failed to load pbs config"); + initialize_tracing_log(pbs_config.logs_settings.clone()); let state = PbsState::<()>::new(pbs_config); PbsService::init_metrics()?; diff --git a/bin/signer_module.rs b/bin/signer_module.rs index c1c9ea79..c5dfb420 100644 --- a/bin/signer_module.rs +++ b/bin/signer_module.rs @@ -11,8 +11,7 @@ async fn main() -> Result<()> { std::env::set_var("RUST_BACKTRACE", "1"); } - initialize_tracing_log(); - let config = StartSignerConfig::load_from_env()?; + initialize_tracing_log(config.logs_settings.clone()); SigningService::run(config).await } diff --git a/config.example.toml b/config.example.toml index bfd1add3..fd8b1285 100644 --- a/config.example.toml +++ b/config.example.toml @@ -33,3 +33,7 @@ use_grafana = true id = "DA_COMMIT" docker_image = "test_da_commit" sleep_secs = 5 + +[logs] +duration = "hourly" +file_name_prefix = "pbs" \ No newline at end of file diff --git a/crates/common/src/config.rs b/crates/common/src/config.rs index 24c54f48..bafa62c2 100644 --- a/crates/common/src/config.rs +++ b/crates/common/src/config.rs @@ -41,6 +41,32 @@ pub struct CommitBoostConfig { pub modules: Option>, pub signer: Option, pub metrics: MetricsConfig, + #[serde(default)] + pub logs: LogsSettings, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct LogsSettings { + pub duration: RollingDuration, + pub file_name_prefix: String, +} + +impl Default for LogsSettings { + fn default() -> Self { + Self { duration: RollingDuration::Hourly, file_name_prefix: "".to_string() } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum RollingDuration { + #[serde(rename = "minutely")] + Minutely, + #[serde(rename = "hourly")] + Hourly, + #[serde(rename = "daily")] + Daily, + #[serde(rename = "never")] + Never, } fn load_from_file(path: &str) -> Result { @@ -89,6 +115,7 @@ pub struct StartSignerConfig { pub loader: SignerLoader, pub server_port: u16, pub jwts: HashMap, + pub logs_settings: LogsSettings, } impl StartSignerConfig { @@ -103,6 +130,7 @@ impl StartSignerConfig { loader: config.signer.expect("Signer config is missing").loader, server_port, jwts, + logs_settings: config.logs, }) } } @@ -156,6 +184,9 @@ pub struct PbsModuleConfig { pub signer_client: Option, /// Opaque module config pub extra: T, + /// Settings for logging in file, refer to Default implementation to see + /// default values if not set. + pub logs_settings: LogsSettings, } fn default_pbs() -> String { @@ -174,6 +205,7 @@ pub fn load_pbs_config() -> Result> { relays: relay_clients, signer_client: None, extra: (), + logs_settings: config.logs, }) } @@ -192,6 +224,8 @@ pub fn load_pbs_custom_config() -> Result, pbs: CustomPbsConfig, + #[serde(default)] + logs_settings: LogsSettings, } // load module config including the extra data (if any) @@ -215,6 +249,7 @@ pub fn load_pbs_custom_config() -> Result { pub signer_client: SignerClient, /// Opaque module config pub extra: T, + pub logs_settings: LogsSettings, } /// Loads a module config from the environment and config file: @@ -269,6 +305,8 @@ pub fn load_module_config() -> Result> struct StubConfig { chain: Chain, modules: Vec>, + #[serde(default)] + logs_settings: LogsSettings, } // load module config including the extra data (if any) @@ -296,6 +334,7 @@ pub fn load_module_config() -> Result> chain: cb_config.chain, signer_client, extra: module_config.extra, + logs_settings: cb_config.logs_settings, }) } } diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index b1cc574e..48693586 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -1,7 +1,4 @@ -use std::{ - fmt::Debug, - time::{SystemTime, UNIX_EPOCH}, -}; +use std::time::{SystemTime, UNIX_EPOCH}; use alloy::{ primitives::U256, @@ -10,12 +7,14 @@ use alloy::{ use blst::min_pk::{PublicKey, Signature}; use rand::{distributions::Alphanumeric, Rng}; use reqwest::header::HeaderMap; -use tracing_appender::rolling::{RollingFileAppender, Rotation}; use tracing_subscriber::{ fmt, fmt::writer::MakeWriterExt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, }; -use crate::types::Chain; +use crate::{ + config::{LogsSettings, RollingDuration}, + types::Chain, +}; const SECONDS_PER_SLOT: u64 = 12; const MILLIS_PER_SECOND: u64 = 1_000; @@ -117,10 +116,23 @@ pub const fn default_u256() -> U256 { // LOGGING // TODO: more customized logging + logging guard -pub fn initialize_tracing_log(file_name_prefix: &str) { +pub fn initialize_tracing_log(logs_settings: LogsSettings) { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); // Log all events to a rolling log file. - let logfile = tracing_appender::rolling::hourly(format!("/logs/{file_name_prefix}"), file_name_prefix); + let logfile = match logs_settings.duration { + RollingDuration::Minutely => { + tracing_appender::rolling::minutely("/logs", &logs_settings.file_name_prefix) + } + RollingDuration::Hourly => { + tracing_appender::rolling::hourly("/logs", &logs_settings.file_name_prefix) + } + RollingDuration::Daily => { + tracing_appender::rolling::daily("/logs", &logs_settings.file_name_prefix) + } + RollingDuration::Never => { + tracing_appender::rolling::never("/logs", &logs_settings.file_name_prefix) + } + }; // Log `INFO` and above to stdout. let stdout = std::io::stdout.with_max_level(tracing::Level::INFO); let filter = match level_env.parse::() { diff --git a/examples/da_commit/src/main.rs b/examples/da_commit/src/main.rs index 10dda090..f894f049 100644 --- a/examples/da_commit/src/main.rs +++ b/examples/da_commit/src/main.rs @@ -71,7 +71,6 @@ impl DaCommitService { #[tokio::main] async fn main() -> Result<()> { color_eyre::install()?; - initialize_tracing_log(); // Remember to register all your metrics before starting the process MY_CUSTOM_REGISTRY.register(Box::new(SIG_RECEIVED_COUNTER.clone()))?; @@ -85,7 +84,7 @@ async fn main() -> Result<()> { sleep_secs = config.extra.sleep_secs, "Starting module with custom data" ); - + initialize_tracing_log(config.logs_settings.clone()); let service = DaCommitService { config }; if let Err(err) = service.run().await { diff --git a/tests/tests/pbs_integration.rs b/tests/tests/pbs_integration.rs index a3d089e0..943d6b5f 100644 --- a/tests/tests/pbs_integration.rs +++ b/tests/tests/pbs_integration.rs @@ -52,6 +52,7 @@ fn to_pbs_config( signer_client: None, extra: (), relays, + logs_settings: Default::default(), } } From 2fcee1f3d59f2a355a01600cbeddefec1616975e Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 10:27:52 +0100 Subject: [PATCH 04/22] add logs.prefixes --- bin/default_pbs.rs | 2 +- bin/signer_module.rs | 2 +- config.example.toml | 6 +++++- crates/common/src/config.rs | 4 ++-- crates/common/src/utils.rs | 22 +++++++++------------- examples/da_commit/src/main.rs | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bin/default_pbs.rs b/bin/default_pbs.rs index 48de6db5..155ff4c8 100644 --- a/bin/default_pbs.rs +++ b/bin/default_pbs.rs @@ -13,7 +13,7 @@ async fn main() -> Result<()> { // TODO: handle errors let pbs_config = load_pbs_config().expect("failed to load pbs config"); - initialize_tracing_log(pbs_config.logs_settings.clone()); + initialize_tracing_log(pbs_config.logs_settings.clone(), "pbs"); let state = PbsState::<()>::new(pbs_config); PbsService::init_metrics()?; diff --git a/bin/signer_module.rs b/bin/signer_module.rs index c5dfb420..2e27280c 100644 --- a/bin/signer_module.rs +++ b/bin/signer_module.rs @@ -12,6 +12,6 @@ async fn main() -> Result<()> { } let config = StartSignerConfig::load_from_env()?; - initialize_tracing_log(config.logs_settings.clone()); + initialize_tracing_log(config.logs_settings.clone(), "signer"); SigningService::run(config).await } diff --git a/config.example.toml b/config.example.toml index fd8b1285..34f0d89e 100644 --- a/config.example.toml +++ b/config.example.toml @@ -36,4 +36,8 @@ sleep_secs = 5 [logs] duration = "hourly" -file_name_prefix = "pbs" \ No newline at end of file + +[logs.prefixes] +pbs="pbs" +signer="signer" +da_commit="da" \ No newline at end of file diff --git a/crates/common/src/config.rs b/crates/common/src/config.rs index bafa62c2..7cf6b711 100644 --- a/crates/common/src/config.rs +++ b/crates/common/src/config.rs @@ -48,12 +48,12 @@ pub struct CommitBoostConfig { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct LogsSettings { pub duration: RollingDuration, - pub file_name_prefix: String, + pub prefixes: HashMap, } impl Default for LogsSettings { fn default() -> Self { - Self { duration: RollingDuration::Hourly, file_name_prefix: "".to_string() } + Self { duration: RollingDuration::Hourly, prefixes: Default::default() } } } diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 48693586..8be81304 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -116,22 +116,18 @@ pub const fn default_u256() -> U256 { // LOGGING // TODO: more customized logging + logging guard -pub fn initialize_tracing_log(logs_settings: LogsSettings) { +pub fn initialize_tracing_log(logs_settings: LogsSettings, module_name: &str) { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let prefix = logs_settings + .prefixes + .get(module_name) + .unwrap_or_else(|| panic!("prefix not found for module name {module_name}")); // Log all events to a rolling log file. let logfile = match logs_settings.duration { - RollingDuration::Minutely => { - tracing_appender::rolling::minutely("/logs", &logs_settings.file_name_prefix) - } - RollingDuration::Hourly => { - tracing_appender::rolling::hourly("/logs", &logs_settings.file_name_prefix) - } - RollingDuration::Daily => { - tracing_appender::rolling::daily("/logs", &logs_settings.file_name_prefix) - } - RollingDuration::Never => { - tracing_appender::rolling::never("/logs", &logs_settings.file_name_prefix) - } + RollingDuration::Minutely => tracing_appender::rolling::minutely("/var/logs", prefix), + RollingDuration::Hourly => tracing_appender::rolling::hourly("/var/logs", prefix), + RollingDuration::Daily => tracing_appender::rolling::daily("/var/logs", prefix), + RollingDuration::Never => tracing_appender::rolling::never("/var/logs", prefix), }; // Log `INFO` and above to stdout. let stdout = std::io::stdout.with_max_level(tracing::Level::INFO); diff --git a/examples/da_commit/src/main.rs b/examples/da_commit/src/main.rs index f894f049..d60a6b7a 100644 --- a/examples/da_commit/src/main.rs +++ b/examples/da_commit/src/main.rs @@ -84,7 +84,7 @@ async fn main() -> Result<()> { sleep_secs = config.extra.sleep_secs, "Starting module with custom data" ); - initialize_tracing_log(config.logs_settings.clone()); + initialize_tracing_log(config.logs_settings.clone(), "da_commit"); let service = DaCommitService { config }; if let Err(err) = service.run().await { From 79d6806a4606cbd572d3e57789e86058681aec73 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 10:42:38 +0100 Subject: [PATCH 05/22] fmt --- crates/cli/src/docker_init.rs | 9 ++++++++- crates/common/src/config/mod.rs | 2 +- crates/common/src/config/module.rs | 3 +-- examples/builder_log/src/main.rs | 2 -- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index adc98ad5..d0b5a7c7 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -288,7 +288,14 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), depends_on: DependsOnOptions::Simple(vec!["cb_prometheus".to_owned()]), environment: Environment::List(vec!["GF_SECURITY_ADMIN_PASSWORD=admin".to_owned()]), - volumes: vec![Volumes::Simple("./grafana/dashboards:/etc/grafana/provisioning/dashboards".to_owned()), Volumes::Simple("./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned())], + volumes: vec![ + Volumes::Simple( + "./grafana/dashboards:/etc/grafana/provisioning/dashboards".to_owned(), + ), + Volumes::Simple( + "./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned(), + ), + ], // TODO: re-enable logging here once we move away from docker logs logging: Some(LoggingParameters { driver: Some("none".to_owned()), options: None }), ..Service::default() diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 6478dea5..a427f8a4 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -64,4 +64,4 @@ pub enum RollingDuration { Daily, #[serde(rename = "never")] Never, -} \ No newline at end of file +} diff --git a/crates/common/src/config/module.rs b/crates/common/src/config/module.rs index 35d7672a..e9c8fc3e 100644 --- a/crates/common/src/config/module.rs +++ b/crates/common/src/config/module.rs @@ -2,6 +2,7 @@ use eyre::{ContextCompat, Result}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use toml::Table; +use super::LogsSettings; use crate::{ commit::client::SignerClient, config::{ @@ -13,8 +14,6 @@ use crate::{ types::Chain, }; -use super::LogsSettings; - #[derive(Debug, Deserialize, Serialize)] pub enum ModuleKind { #[serde(alias = "commit")] diff --git a/examples/builder_log/src/main.rs b/examples/builder_log/src/main.rs index acfd36a6..1d34ce99 100644 --- a/examples/builder_log/src/main.rs +++ b/examples/builder_log/src/main.rs @@ -14,8 +14,6 @@ impl OnBuilderApiEvent for LogProcessor { #[tokio::main] async fn main() { - - match load_builder_module_config::<()>() { Ok(config) => { info!(module_id = config.id, "Starting module"); From 894a11550fb3147dd9c1d0f3421860f6f4d5cd17 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 11:21:41 +0100 Subject: [PATCH 06/22] move to logs --- crates/common/src/config/log.rs | 23 +++++++++++++++++++++++ crates/common/src/config/mod.rs | 31 ++++--------------------------- 2 files changed, 27 insertions(+), 27 deletions(-) create mode 100644 crates/common/src/config/log.rs diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs new file mode 100644 index 00000000..6970a1ed --- /dev/null +++ b/crates/common/src/config/log.rs @@ -0,0 +1,23 @@ +use std::collections::HashMap; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct LogsSettings { + pub duration: RollingDuration, + pub prefixes: HashMap, +} + +impl Default for LogsSettings { + fn default() -> Self { + Self { duration: RollingDuration::Hourly, prefixes: Default::default() } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum RollingDuration { + Minutely, + Hourly, + Daily, + Never, +} diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index a427f8a4..2601937a 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use eyre::Result; use serde::{Deserialize, Serialize}; @@ -12,12 +10,15 @@ mod pbs; mod signer; mod utils; +mod log; + pub use constants::*; pub use metrics::*; pub use module::*; pub use pbs::*; pub use signer::*; pub use utils::*; +pub use log::*; #[derive(Debug, Deserialize, Serialize)] pub struct CommitBoostConfig { @@ -40,28 +41,4 @@ impl CommitBoostConfig { pub fn from_env_path() -> Result { load_file_from_env(CB_CONFIG_ENV) } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct LogsSettings { - pub duration: RollingDuration, - pub prefixes: HashMap, -} - -impl Default for LogsSettings { - fn default() -> Self { - Self { duration: RollingDuration::Hourly, prefixes: Default::default() } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub enum RollingDuration { - #[serde(rename = "minutely")] - Minutely, - #[serde(rename = "hourly")] - Hourly, - #[serde(rename = "daily")] - Daily, - #[serde(rename = "never")] - Never, -} +} \ No newline at end of file From c19f78846f113e61eedaeb305e2ed02c2665b56c Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 11:36:13 +0100 Subject: [PATCH 07/22] comments --- crates/common/src/config/log.rs | 15 ++++++++++++--- crates/common/src/config/mod.rs | 4 ++-- crates/common/src/utils.rs | 22 +++++++++++++--------- examples/da_commit/src/main.rs | 2 +- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs index 6970a1ed..ddeb8fd5 100644 --- a/crates/common/src/config/log.rs +++ b/crates/common/src/config/log.rs @@ -1,15 +1,18 @@ -use std::collections::HashMap; +use std::path::PathBuf; + use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct LogsSettings { + #[serde(default)] pub duration: RollingDuration, - pub prefixes: HashMap, + #[serde(default)] + pub base_path: PathBuf, } impl Default for LogsSettings { fn default() -> Self { - Self { duration: RollingDuration::Hourly, prefixes: Default::default() } + Self { duration: RollingDuration::Hourly, base_path: Default::default() } } } @@ -21,3 +24,9 @@ pub enum RollingDuration { Daily, Never, } + +impl Default for RollingDuration { + fn default() -> Self { + Self::Daily + } +} diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 2601937a..a3ece576 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -13,12 +13,12 @@ mod utils; mod log; pub use constants::*; +pub use log::*; pub use metrics::*; pub use module::*; pub use pbs::*; pub use signer::*; pub use utils::*; -pub use log::*; #[derive(Debug, Deserialize, Serialize)] pub struct CommitBoostConfig { @@ -41,4 +41,4 @@ impl CommitBoostConfig { pub fn from_env_path() -> Result { load_file_from_env(CB_CONFIG_ENV) } -} \ No newline at end of file +} diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 8be81304..c1376327 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -116,18 +116,22 @@ pub const fn default_u256() -> U256 { // LOGGING // TODO: more customized logging + logging guard -pub fn initialize_tracing_log(logs_settings: LogsSettings, module_name: &str) { +pub fn initialize_tracing_log(logs_settings: LogsSettings, module_id: &str) { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); - let prefix = logs_settings - .prefixes - .get(module_name) - .unwrap_or_else(|| panic!("prefix not found for module name {module_name}")); // Log all events to a rolling log file. let logfile = match logs_settings.duration { - RollingDuration::Minutely => tracing_appender::rolling::minutely("/var/logs", prefix), - RollingDuration::Hourly => tracing_appender::rolling::hourly("/var/logs", prefix), - RollingDuration::Daily => tracing_appender::rolling::daily("/var/logs", prefix), - RollingDuration::Never => tracing_appender::rolling::never("/var/logs", prefix), + RollingDuration::Minutely => { + tracing_appender::rolling::minutely(logs_settings.base_path, module_id) + } + RollingDuration::Hourly => { + tracing_appender::rolling::hourly(logs_settings.base_path, module_id) + } + RollingDuration::Daily => { + tracing_appender::rolling::daily(logs_settings.base_path, module_id) + } + RollingDuration::Never => { + tracing_appender::rolling::never(logs_settings.base_path, module_id) + } }; // Log `INFO` and above to stdout. let stdout = std::io::stdout.with_max_level(tracing::Level::INFO); diff --git a/examples/da_commit/src/main.rs b/examples/da_commit/src/main.rs index 09ab32cb..ac8f0ac1 100644 --- a/examples/da_commit/src/main.rs +++ b/examples/da_commit/src/main.rs @@ -84,7 +84,7 @@ async fn main() -> Result<()> { sleep_secs = config.extra.sleep_secs, "Starting module with custom data" ); - initialize_tracing_log(config.logs_settings.clone(), "da_commit"); + initialize_tracing_log(config.logs_settings.clone(), &config.id); let service = DaCommitService { config }; if let Err(err) = service.run().await { From e7859bafa114483f4cd52cebce45b1a551f00aea Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 13:16:54 +0100 Subject: [PATCH 08/22] utils --- crates/common/src/utils.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index c1376327..ae00838e 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -1,4 +1,7 @@ -use std::time::{SystemTime, UNIX_EPOCH}; +use std::{ + str::FromStr, + time::{SystemTime, UNIX_EPOCH}, +}; use alloy::{ primitives::U256, @@ -7,9 +10,7 @@ use alloy::{ use blst::min_pk::{PublicKey, Signature}; use rand::{distributions::Alphanumeric, Rng}; use reqwest::header::HeaderMap; -use tracing_subscriber::{ - fmt, fmt::writer::MakeWriterExt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, -}; +use tracing_subscriber::{fmt::Layer, prelude::*, EnvFilter}; use crate::{ config::{LogsSettings, RollingDuration}, @@ -119,7 +120,7 @@ pub const fn default_u256() -> U256 { pub fn initialize_tracing_log(logs_settings: LogsSettings, module_id: &str) { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); // Log all events to a rolling log file. - let logfile = match logs_settings.duration { + let log_file = match logs_settings.duration { RollingDuration::Minutely => { tracing_appender::rolling::minutely(logs_settings.base_path, module_id) } @@ -133,8 +134,7 @@ pub fn initialize_tracing_log(logs_settings: LogsSettings, module_id: &str) { tracing_appender::rolling::never(logs_settings.base_path, module_id) } }; - // Log `INFO` and above to stdout. - let stdout = std::io::stdout.with_max_level(tracing::Level::INFO); + let filter = match level_env.parse::() { Ok(f) => f, Err(_) => { @@ -142,12 +142,13 @@ pub fn initialize_tracing_log(logs_settings: LogsSettings, module_id: &str) { EnvFilter::new("info") } }; + let logging_level = tracing::Level::from_str(&level_env) + .unwrap_or_else(|_| panic!("invalid value for tracing. Got {level_env}")); + let stdout_log = tracing_subscriber::fmt::layer().pretty(); + let (default, _guard) = tracing_appender::non_blocking(log_file); + let log_file = Layer::new().with_writer(default.with_max_level(logging_level)); - tracing_subscriber::registry() - .with(filter) - .with(fmt::layer().with_writer(stdout.and(logfile))) - .try_init() - .unwrap(); + tracing_subscriber::registry().with(stdout_log.with_filter(filter).and_then(log_file)).init(); } pub fn print_logo() { From b9dda753d913f4ad54d2b192f7388c52b6e25ccc Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 13:54:55 +0100 Subject: [PATCH 09/22] handle guard --- Cargo.toml | 1 + bin/default_pbs.rs | 3 +-- bin/signer_module.rs | 2 +- crates/common/src/utils.rs | 8 +++++--- examples/builder_log/Cargo.toml | 3 +++ examples/builder_log/src/main.rs | 2 +- examples/custom_boost.rs | 2 +- examples/da_commit/Cargo.toml | 2 ++ examples/da_commit/src/main.rs | 2 +- 9 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 279fdb12..1b88e4d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ docker-compose-types = "0.12.0" bollard = "0.16.1" # misc +backtrace = "0.3.67" clap = { version = "4.5.4", features = ["derive", "env"] } thiserror = "1.0.61" color-eyre = "0.6.3" diff --git a/bin/default_pbs.rs b/bin/default_pbs.rs index 155ff4c8..0de3eacf 100644 --- a/bin/default_pbs.rs +++ b/bin/default_pbs.rs @@ -13,9 +13,8 @@ async fn main() -> Result<()> { // TODO: handle errors let pbs_config = load_pbs_config().expect("failed to load pbs config"); - initialize_tracing_log(pbs_config.logs_settings.clone(), "pbs"); + let _guard = initialize_tracing_log(pbs_config.logs_settings.clone(), "pbs"); let state = PbsState::<()>::new(pbs_config); - PbsService::init_metrics()?; PbsService::run::<(), DefaultBuilderApi>(state).await; Ok(()) diff --git a/bin/signer_module.rs b/bin/signer_module.rs index 2e27280c..023cfe27 100644 --- a/bin/signer_module.rs +++ b/bin/signer_module.rs @@ -12,6 +12,6 @@ async fn main() -> Result<()> { } let config = StartSignerConfig::load_from_env()?; - initialize_tracing_log(config.logs_settings.clone(), "signer"); + let _guard = initialize_tracing_log(config.logs_settings.clone(), "signer"); SigningService::run(config).await } diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index ae00838e..6649d1d5 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -10,6 +10,7 @@ use alloy::{ use blst::min_pk::{PublicKey, Signature}; use rand::{distributions::Alphanumeric, Rng}; use reqwest::header::HeaderMap; +use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{fmt::Layer, prelude::*, EnvFilter}; use crate::{ @@ -116,8 +117,7 @@ pub const fn default_u256() -> U256 { } // LOGGING -// TODO: more customized logging + logging guard -pub fn initialize_tracing_log(logs_settings: LogsSettings, module_id: &str) { +pub fn initialize_tracing_log(logs_settings: LogsSettings, module_id: &str) -> WorkerGuard { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); // Log all events to a rolling log file. let log_file = match logs_settings.duration { @@ -145,10 +145,12 @@ pub fn initialize_tracing_log(logs_settings: LogsSettings, module_id: &str) { let logging_level = tracing::Level::from_str(&level_env) .unwrap_or_else(|_| panic!("invalid value for tracing. Got {level_env}")); let stdout_log = tracing_subscriber::fmt::layer().pretty(); - let (default, _guard) = tracing_appender::non_blocking(log_file); + let (default, guard) = tracing_appender::non_blocking(log_file); let log_file = Layer::new().with_writer(default.with_max_level(logging_level)); tracing_subscriber::registry().with(stdout_log.with_filter(filter).and_then(log_file)).init(); + + guard } pub fn print_logo() { diff --git a/examples/builder_log/Cargo.toml b/examples/builder_log/Cargo.toml index ee5b2780..6fd87484 100644 --- a/examples/builder_log/Cargo.toml +++ b/examples/builder_log/Cargo.toml @@ -6,6 +6,7 @@ rust-version.workspace = true [dependencies] commit-boost = { path = "../../bin" } +cb-common = { path = "../../crates/common" } # networking reqwest.workspace = true @@ -19,3 +20,5 @@ serde.workspace = true # telemetry tracing.workspace = true + +backtrace.workspace = true diff --git a/examples/builder_log/src/main.rs b/examples/builder_log/src/main.rs index 1d34ce99..2d18dfc8 100644 --- a/examples/builder_log/src/main.rs +++ b/examples/builder_log/src/main.rs @@ -17,7 +17,7 @@ async fn main() { match load_builder_module_config::<()>() { Ok(config) => { info!(module_id = config.id, "Starting module"); - initialize_tracing_log(config.logs_settings.clone(), "logs"); + let _guard = initialize_tracing_log(config.logs_settings.clone(), "builder_log"); let client = BuilderEventClient::new(config.server_port, LogProcessor); if let Err(err) = client.run().await { diff --git a/examples/custom_boost.rs b/examples/custom_boost.rs index d2587fdb..e3ce7131 100644 --- a/examples/custom_boost.rs +++ b/examples/custom_boost.rs @@ -58,7 +58,7 @@ async fn handle_stats(State(state): State>) -> Respo async fn main() { color_eyre::install()?; - initialize_tracing_log(); + let _guard = initialize_tracing_log(); let (chain, config) = load_pbs_config(); diff --git a/examples/da_commit/Cargo.toml b/examples/da_commit/Cargo.toml index 9ad0fe28..d3cd4ca0 100644 --- a/examples/da_commit/Cargo.toml +++ b/examples/da_commit/Cargo.toml @@ -6,6 +6,7 @@ rust-version.workspace = true [dependencies] commit-boost = { path = "../../bin" } +cb-common = { path = "../../crates/common" } # ethereum alloy.workspace = true @@ -24,6 +25,7 @@ tracing.workspace = true prometheus.workspace = true # misc +backtrace.workspace = true eyre.workspace = true color-eyre.workspace = true lazy_static.workspace = true diff --git a/examples/da_commit/src/main.rs b/examples/da_commit/src/main.rs index ac8f0ac1..8d8a4119 100644 --- a/examples/da_commit/src/main.rs +++ b/examples/da_commit/src/main.rs @@ -84,7 +84,7 @@ async fn main() -> Result<()> { sleep_secs = config.extra.sleep_secs, "Starting module with custom data" ); - initialize_tracing_log(config.logs_settings.clone(), &config.id); + let _guard = initialize_tracing_log(config.logs_settings.clone(), &config.id); let service = DaCommitService { config }; if let Err(err) = service.run().await { From fa63571e650488fff95187ff945415d38508f37a Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 13:56:08 +0100 Subject: [PATCH 10/22] remove unused deps --- Cargo.toml | 1 - examples/builder_log/Cargo.toml | 2 -- examples/da_commit/Cargo.toml | 2 -- 3 files changed, 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1b88e4d7..279fdb12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,6 @@ docker-compose-types = "0.12.0" bollard = "0.16.1" # misc -backtrace = "0.3.67" clap = { version = "4.5.4", features = ["derive", "env"] } thiserror = "1.0.61" color-eyre = "0.6.3" diff --git a/examples/builder_log/Cargo.toml b/examples/builder_log/Cargo.toml index 6fd87484..4011210e 100644 --- a/examples/builder_log/Cargo.toml +++ b/examples/builder_log/Cargo.toml @@ -6,7 +6,6 @@ rust-version.workspace = true [dependencies] commit-boost = { path = "../../bin" } -cb-common = { path = "../../crates/common" } # networking reqwest.workspace = true @@ -21,4 +20,3 @@ serde.workspace = true # telemetry tracing.workspace = true -backtrace.workspace = true diff --git a/examples/da_commit/Cargo.toml b/examples/da_commit/Cargo.toml index d3cd4ca0..9ad0fe28 100644 --- a/examples/da_commit/Cargo.toml +++ b/examples/da_commit/Cargo.toml @@ -6,7 +6,6 @@ rust-version.workspace = true [dependencies] commit-boost = { path = "../../bin" } -cb-common = { path = "../../crates/common" } # ethereum alloy.workspace = true @@ -25,7 +24,6 @@ tracing.workspace = true prometheus.workspace = true # misc -backtrace.workspace = true eyre.workspace = true color-eyre.workspace = true lazy_static.workspace = true From 47a8ac8f932afcc4d2e9a82c4e1100bdaebffbfe Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 13:59:05 +0100 Subject: [PATCH 11/22] default log location if not specified in configuration --- crates/common/src/config/log.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs index ddeb8fd5..8999c401 100644 --- a/crates/common/src/config/log.rs +++ b/crates/common/src/config/log.rs @@ -12,7 +12,7 @@ pub struct LogsSettings { impl Default for LogsSettings { fn default() -> Self { - Self { duration: RollingDuration::Hourly, base_path: Default::default() } + Self { duration: RollingDuration::Hourly, base_path: "/var/logs".into() } } } From c55d42f98ebebd72f4556fe998e25a08139235df Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 14:47:25 +0100 Subject: [PATCH 12/22] as discussed --- bin/default_pbs.rs | 2 +- bin/signer_module.rs | 2 +- config.example.toml | 7 +------ crates/cli/src/docker_init.rs | 22 +++++++++++++-------- crates/common/src/config/constants.rs | 2 ++ crates/common/src/config/log.rs | 20 ++++++++++++++++--- crates/common/src/config/module.rs | 9 --------- crates/common/src/config/pbs.rs | 7 +------ crates/common/src/config/signer.rs | 4 +--- crates/common/src/utils.rs | 28 ++++++++++----------------- examples/builder_log/src/main.rs | 2 +- examples/custom_boost.rs | 2 +- examples/da_commit/src/main.rs | 2 +- scripts/build_local_images.sh | 0 scripts/build_local_modules.sh | 0 tests/tests/pbs_integration.rs | 1 - 16 files changed, 51 insertions(+), 59 deletions(-) mode change 100644 => 100755 scripts/build_local_images.sh mode change 100644 => 100755 scripts/build_local_modules.sh diff --git a/bin/default_pbs.rs b/bin/default_pbs.rs index 0de3eacf..96b43a64 100644 --- a/bin/default_pbs.rs +++ b/bin/default_pbs.rs @@ -13,7 +13,7 @@ async fn main() -> Result<()> { // TODO: handle errors let pbs_config = load_pbs_config().expect("failed to load pbs config"); - let _guard = initialize_tracing_log(pbs_config.logs_settings.clone(), "pbs"); + let _guard = initialize_tracing_log("pbs"); let state = PbsState::<()>::new(pbs_config); PbsService::init_metrics()?; PbsService::run::<(), DefaultBuilderApi>(state).await; diff --git a/bin/signer_module.rs b/bin/signer_module.rs index 023cfe27..59d84671 100644 --- a/bin/signer_module.rs +++ b/bin/signer_module.rs @@ -12,6 +12,6 @@ async fn main() -> Result<()> { } let config = StartSignerConfig::load_from_env()?; - let _guard = initialize_tracing_log(config.logs_settings.clone(), "signer"); + let _guard = initialize_tracing_log("signer"); SigningService::run(config).await } diff --git a/config.example.toml b/config.example.toml index f70f5749..8e754747 100644 --- a/config.example.toml +++ b/config.example.toml @@ -42,9 +42,4 @@ docker_image = "test_builder_log" [logs] duration = "hourly" - -[logs.prefixes] -pbs="pbs" -signer="signer" -da_commit="da" -logs="logs" +host_path="./logs" \ No newline at end of file diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index d0b5a7c7..443d07b1 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -2,10 +2,10 @@ use std::{path::Path, vec}; use cb_common::{ config::{ - CommitBoostConfig, ModuleKind, BUILDER_SERVER_ENV, CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV, - METRICS_SERVER_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV, - SIGNER_DIR_SECRETS, SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, SIGNER_KEYS_ENV, - SIGNER_SERVER_ENV, + CommitBoostConfig, ModuleKind, BUILDER_SERVER_ENV, CB_BASE_LOG_PATH, CB_CONFIG_ENV, + CB_CONFIG_NAME, JWTS_ENV, METRICS_SERVER_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, + SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS, SIGNER_DIR_SECRETS_ENV, + SIGNER_KEYS, SIGNER_KEYS_ENV, SIGNER_SERVER_ENV, }, loader::SignerLoader, utils::random_jwt, @@ -41,10 +41,16 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> // config volume to pass to all services let config_volume = Volumes::Simple(format!("./{}:{}:ro", config_path, CB_CONFIG_NAME)); + let log_volume = Volumes::Simple(format!( + "{}:{}", + cb_config.logs.host_path.to_str().unwrap(), + CB_BASE_LOG_PATH + )); let mut jwts = IndexMap::new(); // envs to write in .env file let mut envs = IndexMap::from([(CB_CONFIG_ENV.into(), CB_CONFIG_NAME.into())]); + envs.insert("ROLLING_DURATION".into(), cb_config.logs.duration.to_string()); // targets to pass to prometheus let mut targets = Vec::new(); @@ -109,7 +115,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> METRICS_NETWORK.to_owned(), SIGNER_NETWORK.to_owned(), ]), - volumes: vec![config_volume.clone()], + volumes: vec![config_volume.clone(), log_volume.clone()], environment: Environment::KvPair(module_envs), depends_on: DependsOnOptions::Simple(vec!["cb_signer".to_owned()]), ..Service::default() @@ -131,7 +137,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> container_name: Some(module_cid.clone()), image: Some(module.docker_image), networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - volumes: vec![config_volume.clone()], + volumes: vec![config_volume.clone(), log_volume.clone()], environment: Environment::KvPair(module_envs), depends_on: DependsOnOptions::Simple(vec!["cb_pbs".to_owned()]), ..Service::default() @@ -157,7 +163,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> cb_config.pbs.pbs_config.port, cb_config.pbs.pbs_config.port )]), networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - volumes: vec![config_volume.clone()], + volumes: vec![config_volume.clone(), log_volume.clone()], environment: Environment::KvPair(pbs_envs), ..Service::default() }; @@ -170,7 +176,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> if let Some(signer_config) = cb_config.signer { if needs_signer_module { - let mut volumes = vec![config_volume.clone()]; + let mut volumes = vec![config_volume.clone(), log_volume.clone()]; targets.push(PrometheusTargetConfig { targets: vec![format!("cb_signer:{metrics_port}")], diff --git a/crates/common/src/config/constants.rs b/crates/common/src/config/constants.rs index 4d96d80f..ade25fa8 100644 --- a/crates/common/src/config/constants.rs +++ b/crates/common/src/config/constants.rs @@ -4,6 +4,8 @@ pub const METRICS_SERVER_ENV: &str = "METRICS_SERVER"; pub const SIGNER_SERVER_ENV: &str = "SIGNER_SERVER"; pub const BUILDER_SERVER_ENV: &str = "BUILDER_SERVER"; +pub const CB_BASE_LOG_PATH: &str = "/var/logs/"; + pub const CB_CONFIG_ENV: &str = "CB_CONFIG"; pub const CB_CONFIG_NAME: &str = "/cb-config.toml"; diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs index 8999c401..8662d0a9 100644 --- a/crates/common/src/config/log.rs +++ b/crates/common/src/config/log.rs @@ -1,4 +1,7 @@ -use std::path::PathBuf; +use std::{ + fmt::{Display, Formatter}, + path::PathBuf, +}; use serde::{Deserialize, Serialize}; @@ -7,12 +10,12 @@ pub struct LogsSettings { #[serde(default)] pub duration: RollingDuration, #[serde(default)] - pub base_path: PathBuf, + pub host_path: PathBuf, } impl Default for LogsSettings { fn default() -> Self { - Self { duration: RollingDuration::Hourly, base_path: "/var/logs".into() } + Self { duration: RollingDuration::Hourly, host_path: "/var/logs".into() } } } @@ -25,6 +28,17 @@ pub enum RollingDuration { Never, } +impl Display for RollingDuration { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + RollingDuration::Minutely => write!(f, "minutely"), + RollingDuration::Hourly => write!(f, "hourly"), + RollingDuration::Daily => write!(f, "daily"), + RollingDuration::Never => write!(f, "never"), + } + } +} + impl Default for RollingDuration { fn default() -> Self { Self::Daily diff --git a/crates/common/src/config/module.rs b/crates/common/src/config/module.rs index e9c8fc3e..508c6b50 100644 --- a/crates/common/src/config/module.rs +++ b/crates/common/src/config/module.rs @@ -2,7 +2,6 @@ use eyre::{ContextCompat, Result}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use toml::Table; -use super::LogsSettings; use crate::{ commit::client::SignerClient, config::{ @@ -45,8 +44,6 @@ pub struct StartCommitModuleConfig { pub signer_client: SignerClient, /// Opaque module config pub extra: T, - - pub logs_settings: LogsSettings, } /// Loads a module config from the environment and config file: @@ -79,7 +76,6 @@ pub fn load_commit_module_config() -> Result { chain: Chain, modules: Vec>, - logs: LogsSettings, } // load module config including the extra data (if any) @@ -109,7 +105,6 @@ pub fn load_commit_module_config() -> Result { pub server_port: u16, /// Opaque module config pub extra: T, - - pub logs_settings: LogsSettings, } pub fn load_builder_module_config() -> eyre::Result> @@ -152,7 +145,6 @@ pub fn load_builder_module_config() -> eyre::Result { chain: Chain, modules: Vec>, - logs: LogsSettings, } // load module config including the extra data (if any) @@ -180,6 +172,5 @@ pub fn load_builder_module_config() -> eyre::Result { pub event_publiher: Option, /// Opaque module config pub extra: T, - - pub logs_settings: LogsSettings, } fn default_pbs() -> String { @@ -112,7 +110,6 @@ pub fn load_pbs_config() -> Result> { signer_client: None, event_publiher: maybe_publiher, extra: (), - logs_settings: config.logs, }) } @@ -131,7 +128,6 @@ pub fn load_pbs_custom_config() -> Result, pbs: CustomPbsConfig, - logs: LogsSettings, } // load module config including the extra data (if any) @@ -156,6 +152,5 @@ pub fn load_pbs_custom_config() -> Result, - pub logs_settings: LogsSettings, } impl StartSignerConfig { @@ -44,7 +43,6 @@ impl StartSignerConfig { loader: config.signer.expect("Signer config is missing").loader, server_port, jwts, - logs_settings: config.logs, }) } } diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 6649d1d5..9fac64a4 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -1,4 +1,5 @@ use std::{ + env, str::FromStr, time::{SystemTime, UNIX_EPOCH}, }; @@ -13,10 +14,7 @@ use reqwest::header::HeaderMap; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{fmt::Layer, prelude::*, EnvFilter}; -use crate::{ - config::{LogsSettings, RollingDuration}, - types::Chain, -}; +use crate::{config::CB_BASE_LOG_PATH, types::Chain}; const SECONDS_PER_SLOT: u64 = 12; const MILLIS_PER_SECOND: u64 = 1_000; @@ -117,22 +115,16 @@ pub const fn default_u256() -> U256 { } // LOGGING -pub fn initialize_tracing_log(logs_settings: LogsSettings, module_id: &str) -> WorkerGuard { +pub fn initialize_tracing_log(module_id: &str) -> WorkerGuard { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); // Log all events to a rolling log file. - let log_file = match logs_settings.duration { - RollingDuration::Minutely => { - tracing_appender::rolling::minutely(logs_settings.base_path, module_id) - } - RollingDuration::Hourly => { - tracing_appender::rolling::hourly(logs_settings.base_path, module_id) - } - RollingDuration::Daily => { - tracing_appender::rolling::daily(logs_settings.base_path, module_id) - } - RollingDuration::Never => { - tracing_appender::rolling::never(logs_settings.base_path, module_id) - } + + let log_file = match env::var("ROLLING_DURATION").unwrap_or("daily".into()).as_str() { + "minutely" => tracing_appender::rolling::minutely(CB_BASE_LOG_PATH, module_id), + "hourly" => tracing_appender::rolling::hourly(CB_BASE_LOG_PATH, module_id), + "daily" => tracing_appender::rolling::daily(CB_BASE_LOG_PATH, module_id), + "never" => tracing_appender::rolling::never(CB_BASE_LOG_PATH, module_id), + _ => panic!("unknown rolling duration value"), }; let filter = match level_env.parse::() { diff --git a/examples/builder_log/src/main.rs b/examples/builder_log/src/main.rs index 2d18dfc8..8b11c225 100644 --- a/examples/builder_log/src/main.rs +++ b/examples/builder_log/src/main.rs @@ -17,7 +17,7 @@ async fn main() { match load_builder_module_config::<()>() { Ok(config) => { info!(module_id = config.id, "Starting module"); - let _guard = initialize_tracing_log(config.logs_settings.clone(), "builder_log"); + let _guard = initialize_tracing_log("builder_log"); let client = BuilderEventClient::new(config.server_port, LogProcessor); if let Err(err) = client.run().await { diff --git a/examples/custom_boost.rs b/examples/custom_boost.rs index e3ce7131..d6ea6d9f 100644 --- a/examples/custom_boost.rs +++ b/examples/custom_boost.rs @@ -58,7 +58,7 @@ async fn handle_stats(State(state): State>) -> Respo async fn main() { color_eyre::install()?; - let _guard = initialize_tracing_log(); + let _guard = initialize_tracing_log("custom_boost"); let (chain, config) = load_pbs_config(); diff --git a/examples/da_commit/src/main.rs b/examples/da_commit/src/main.rs index 8d8a4119..567c46cc 100644 --- a/examples/da_commit/src/main.rs +++ b/examples/da_commit/src/main.rs @@ -84,7 +84,7 @@ async fn main() -> Result<()> { sleep_secs = config.extra.sleep_secs, "Starting module with custom data" ); - let _guard = initialize_tracing_log(config.logs_settings.clone(), &config.id); + let _guard = initialize_tracing_log(&config.id); let service = DaCommitService { config }; if let Err(err) = service.run().await { diff --git a/scripts/build_local_images.sh b/scripts/build_local_images.sh old mode 100644 new mode 100755 diff --git a/scripts/build_local_modules.sh b/scripts/build_local_modules.sh old mode 100644 new mode 100755 diff --git a/tests/tests/pbs_integration.rs b/tests/tests/pbs_integration.rs index 8dc640e3..a0d452d0 100644 --- a/tests/tests/pbs_integration.rs +++ b/tests/tests/pbs_integration.rs @@ -53,7 +53,6 @@ fn to_pbs_config( event_publiher: None, extra: (), relays, - logs_settings: Default::default(), } } From 9ebe941a6e5240e90cb2734c03def199b9bd69e3 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:01:29 +0100 Subject: [PATCH 13/22] constants --- bin/default_pbs.rs | 6 ++++-- bin/signer_module.rs | 6 ++++-- crates/cli/src/docker_init.rs | 6 ++++-- crates/common/src/config/log.rs | 8 +++++++- crates/common/src/lib.rs | 1 + crates/common/src/module_names.rs | 2 ++ 6 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 crates/common/src/module_names.rs diff --git a/bin/default_pbs.rs b/bin/default_pbs.rs index 96b43a64..09b24f23 100644 --- a/bin/default_pbs.rs +++ b/bin/default_pbs.rs @@ -1,4 +1,6 @@ -use cb_common::{config::load_pbs_config, utils::initialize_tracing_log}; +use cb_common::{ + config::load_pbs_config, module_names::PBS_MODULE_NAME, utils::initialize_tracing_log, +}; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use eyre::Result; @@ -13,7 +15,7 @@ async fn main() -> Result<()> { // TODO: handle errors let pbs_config = load_pbs_config().expect("failed to load pbs config"); - let _guard = initialize_tracing_log("pbs"); + let _guard = initialize_tracing_log(PBS_MODULE_NAME); let state = PbsState::<()>::new(pbs_config); PbsService::init_metrics()?; PbsService::run::<(), DefaultBuilderApi>(state).await; diff --git a/bin/signer_module.rs b/bin/signer_module.rs index 59d84671..6d0f328b 100644 --- a/bin/signer_module.rs +++ b/bin/signer_module.rs @@ -1,4 +1,6 @@ -use cb_common::{config::StartSignerConfig, utils::initialize_tracing_log}; +use cb_common::{ + config::StartSignerConfig, module_names::SIGNER_MODULE_NAME, utils::initialize_tracing_log, +}; use cb_signer::service::SigningService; use eyre::Result; @@ -12,6 +14,6 @@ async fn main() -> Result<()> { } let config = StartSignerConfig::load_from_env()?; - let _guard = initialize_tracing_log("signer"); + let _guard = initialize_tracing_log(SIGNER_MODULE_NAME); SigningService::run(config).await } diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 443d07b1..c7af649a 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -24,6 +24,8 @@ pub(super) const CB_ENV_FILE: &str = ".cb.env"; pub(super) const CB_TARGETS_FILE: &str = "targets.json"; // needs to match prometheus.yml pub(super) const PROMETHEUS_DATA_VOLUME: &str = "prometheus-data"; +const ENV_ROLLING_DURATION: &str = "ROLLING_DURATION"; + const METRICS_NETWORK: &str = "monitoring_network"; const SIGNER_NETWORK: &str = "signer_network"; @@ -50,8 +52,8 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let mut jwts = IndexMap::new(); // envs to write in .env file let mut envs = IndexMap::from([(CB_CONFIG_ENV.into(), CB_CONFIG_NAME.into())]); - envs.insert("ROLLING_DURATION".into(), cb_config.logs.duration.to_string()); - + envs.insert(ENV_ROLLING_DURATION.into(), cb_config.logs.duration.to_string()); + envs.insert("RUST_LOG".into(), cb_config.logs.rust_log); // targets to pass to prometheus let mut targets = Vec::new(); let metrics_port = 10000; diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs index 8662d0a9..bf5b3974 100644 --- a/crates/common/src/config/log.rs +++ b/crates/common/src/config/log.rs @@ -11,11 +11,17 @@ pub struct LogsSettings { pub duration: RollingDuration, #[serde(default)] pub host_path: PathBuf, + #[serde(default)] + pub rust_log: String, } impl Default for LogsSettings { fn default() -> Self { - Self { duration: RollingDuration::Hourly, host_path: "/var/logs".into() } + Self { + duration: RollingDuration::Hourly, + host_path: "/var/logs".into(), + rust_log: "info".to_string(), + } } } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 33a44ab4..bc32e50d 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -5,6 +5,7 @@ pub mod config; pub mod constants; pub mod error; pub mod loader; +pub mod module_names; pub mod pbs; pub mod signature; pub mod signer; diff --git a/crates/common/src/module_names.rs b/crates/common/src/module_names.rs new file mode 100644 index 00000000..c6371904 --- /dev/null +++ b/crates/common/src/module_names.rs @@ -0,0 +1,2 @@ +pub const PBS_MODULE_NAME: &str = "str"; +pub const SIGNER_MODULE_NAME: &str = "str"; From d063d0251b773b4d8e8af1d76b6422dd382a0d83 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:03:26 +0100 Subject: [PATCH 14/22] config --- config.example.toml | 3 ++- crates/common/src/config/log.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config.example.toml b/config.example.toml index 8e754747..53c92138 100644 --- a/config.example.toml +++ b/config.example.toml @@ -42,4 +42,5 @@ docker_image = "test_builder_log" [logs] duration = "hourly" -host_path="./logs" \ No newline at end of file +host-path="./logs" +rust-log="info" \ No newline at end of file diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs index bf5b3974..6655eb6d 100644 --- a/crates/common/src/config/log.rs +++ b/crates/common/src/config/log.rs @@ -9,9 +9,9 @@ use serde::{Deserialize, Serialize}; pub struct LogsSettings { #[serde(default)] pub duration: RollingDuration, - #[serde(default)] + #[serde(default, rename="host-path")] pub host_path: PathBuf, - #[serde(default)] + #[serde(default, rename="rust-log")] pub rust_log: String, } From 3922b80e1a34be9b355fe2b3ce360f76ce8c4496 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:04:26 +0100 Subject: [PATCH 15/22] names for modules --- crates/common/src/module_names.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/common/src/module_names.rs b/crates/common/src/module_names.rs index c6371904..54faaa75 100644 --- a/crates/common/src/module_names.rs +++ b/crates/common/src/module_names.rs @@ -1,2 +1,2 @@ -pub const PBS_MODULE_NAME: &str = "str"; -pub const SIGNER_MODULE_NAME: &str = "str"; +pub const PBS_MODULE_NAME: &str = "pbs"; +pub const SIGNER_MODULE_NAME: &str = "signer"; From 9878b85e8871eefb4293925860658dd99c3beda4 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:06:33 +0100 Subject: [PATCH 16/22] rolling duration --- crates/cli/src/docker_init.rs | 3 +-- crates/common/src/utils.rs | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index c7af649a..1bb1ec93 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -17,6 +17,7 @@ use docker_compose_types::{ use eyre::Result; use indexmap::IndexMap; use serde::Serialize; +use cb_common::utils::ENV_ROLLING_DURATION; pub(super) const CB_CONFIG_FILE: &str = "cb-config.toml"; pub(super) const CB_COMPOSE_FILE: &str = "cb.docker-compose.yml"; @@ -24,8 +25,6 @@ pub(super) const CB_ENV_FILE: &str = ".cb.env"; pub(super) const CB_TARGETS_FILE: &str = "targets.json"; // needs to match prometheus.yml pub(super) const PROMETHEUS_DATA_VOLUME: &str = "prometheus-data"; -const ENV_ROLLING_DURATION: &str = "ROLLING_DURATION"; - const METRICS_NETWORK: &str = "monitoring_network"; const SIGNER_NETWORK: &str = "signer_network"; diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 9fac64a4..a325d4ea 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -19,6 +19,8 @@ use crate::{config::CB_BASE_LOG_PATH, types::Chain}; const SECONDS_PER_SLOT: u64 = 12; const MILLIS_PER_SECOND: u64 = 1_000; +pub const ENV_ROLLING_DURATION: &str = "ROLLING_DURATION"; + pub fn timestamp_of_slot_start_millis(slot: u64, chain: Chain) -> u64 { let seconds_since_genesis = chain.genesis_time_sec() + slot * SECONDS_PER_SLOT; seconds_since_genesis * MILLIS_PER_SECOND @@ -119,7 +121,7 @@ pub fn initialize_tracing_log(module_id: &str) -> WorkerGuard { let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); // Log all events to a rolling log file. - let log_file = match env::var("ROLLING_DURATION").unwrap_or("daily".into()).as_str() { + let log_file = match env::var(ENV_ROLLING_DURATION).unwrap_or("daily".into()).as_str() { "minutely" => tracing_appender::rolling::minutely(CB_BASE_LOG_PATH, module_id), "hourly" => tracing_appender::rolling::hourly(CB_BASE_LOG_PATH, module_id), "daily" => tracing_appender::rolling::daily(CB_BASE_LOG_PATH, module_id), From c0adfaf9fd9ee64af3ca51595b6054fb7a29e764 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:07:51 +0100 Subject: [PATCH 17/22] env var --- crates/cli/src/docker_init.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 1bb1ec93..638c3160 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -28,6 +28,8 @@ pub(super) const PROMETHEUS_DATA_VOLUME: &str = "prometheus-data"; const METRICS_NETWORK: &str = "monitoring_network"; const SIGNER_NETWORK: &str = "signer_network"; +const ENV_RUST_LOG: &str = "RUST_LOG"; + /// Builds the docker compose file for the Commit-Boost services // TODO: do more validation for paths, images, etc @@ -52,7 +54,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> // envs to write in .env file let mut envs = IndexMap::from([(CB_CONFIG_ENV.into(), CB_CONFIG_NAME.into())]); envs.insert(ENV_ROLLING_DURATION.into(), cb_config.logs.duration.to_string()); - envs.insert("RUST_LOG".into(), cb_config.logs.rust_log); + envs.insert(ENV_RUST_LOG.into(), cb_config.logs.rust_log); // targets to pass to prometheus let mut targets = Vec::new(); let metrics_port = 10000; From 9b20bfde3eee6dc57694907cec2fc311701f98b4 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:12:34 +0100 Subject: [PATCH 18/22] lint, remove hard coded strings --- crates/cli/src/docker_init.rs | 3 +-- crates/common/src/config/log.rs | 4 ++-- crates/common/src/module_names.rs | 2 ++ examples/builder_log/Cargo.toml | 1 + examples/builder_log/src/main.rs | 3 ++- examples/custom_boost.rs | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 638c3160..1465e699 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -8,7 +8,7 @@ use cb_common::{ SIGNER_KEYS, SIGNER_KEYS_ENV, SIGNER_SERVER_ENV, }, loader::SignerLoader, - utils::random_jwt, + utils::{random_jwt, ENV_ROLLING_DURATION}, }; use docker_compose_types::{ Compose, ComposeVolume, DependsOnOptions, Environment, Labels, LoggingParameters, MapOrEmpty, @@ -17,7 +17,6 @@ use docker_compose_types::{ use eyre::Result; use indexmap::IndexMap; use serde::Serialize; -use cb_common::utils::ENV_ROLLING_DURATION; pub(super) const CB_CONFIG_FILE: &str = "cb-config.toml"; pub(super) const CB_COMPOSE_FILE: &str = "cb.docker-compose.yml"; diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs index 6655eb6d..2cc3f860 100644 --- a/crates/common/src/config/log.rs +++ b/crates/common/src/config/log.rs @@ -9,9 +9,9 @@ use serde::{Deserialize, Serialize}; pub struct LogsSettings { #[serde(default)] pub duration: RollingDuration, - #[serde(default, rename="host-path")] + #[serde(default, rename = "host-path")] pub host_path: PathBuf, - #[serde(default, rename="rust-log")] + #[serde(default, rename = "rust-log")] pub rust_log: String, } diff --git a/crates/common/src/module_names.rs b/crates/common/src/module_names.rs index 54faaa75..f9967262 100644 --- a/crates/common/src/module_names.rs +++ b/crates/common/src/module_names.rs @@ -1,2 +1,4 @@ pub const PBS_MODULE_NAME: &str = "pbs"; pub const SIGNER_MODULE_NAME: &str = "signer"; +pub const BUILDER_LOG_NAME: &str = "builder_log"; +pub const CUSTOM_BOOST_NAME: &str = "custom_boost"; diff --git a/examples/builder_log/Cargo.toml b/examples/builder_log/Cargo.toml index 4011210e..741565f3 100644 --- a/examples/builder_log/Cargo.toml +++ b/examples/builder_log/Cargo.toml @@ -6,6 +6,7 @@ rust-version.workspace = true [dependencies] commit-boost = { path = "../../bin" } +cb-common = {path="../../crates/common"} # networking reqwest.workspace = true diff --git a/examples/builder_log/src/main.rs b/examples/builder_log/src/main.rs index 8b11c225..b58db7c2 100644 --- a/examples/builder_log/src/main.rs +++ b/examples/builder_log/src/main.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use cb_common::module_names::BUILDER_LOG_NAME; use commit_boost::prelude::*; use tracing::{error, info}; @@ -17,7 +18,7 @@ async fn main() { match load_builder_module_config::<()>() { Ok(config) => { info!(module_id = config.id, "Starting module"); - let _guard = initialize_tracing_log("builder_log"); + let _guard = initialize_tracing_log(BUILDER_LOG_NAME); let client = BuilderEventClient::new(config.server_port, LogProcessor); if let Err(err) = client.run().await { diff --git a/examples/custom_boost.rs b/examples/custom_boost.rs index d6ea6d9f..3c408b35 100644 --- a/examples/custom_boost.rs +++ b/examples/custom_boost.rs @@ -58,7 +58,7 @@ async fn handle_stats(State(state): State>) -> Respo async fn main() { color_eyre::install()?; - let _guard = initialize_tracing_log("custom_boost"); + let _guard = initialize_tracing_log(CUSTOM_BOOST_NAME); let (chain, config) = load_pbs_config(); From acab1b19ba73d5ae1bf3f08e259bda01d8f958f5 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:44:05 +0100 Subject: [PATCH 19/22] debug --- crates/common/src/module_names.rs | 2 -- crates/common/src/utils.rs | 8 ++++++-- examples/builder_log/Cargo.toml | 1 - examples/builder_log/src/main.rs | 3 +-- examples/custom_boost.rs | 3 ++- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/crates/common/src/module_names.rs b/crates/common/src/module_names.rs index f9967262..54faaa75 100644 --- a/crates/common/src/module_names.rs +++ b/crates/common/src/module_names.rs @@ -1,4 +1,2 @@ pub const PBS_MODULE_NAME: &str = "pbs"; pub const SIGNER_MODULE_NAME: &str = "signer"; -pub const BUILDER_LOG_NAME: &str = "builder_log"; -pub const CUSTOM_BOOST_NAME: &str = "custom_boost"; diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index a325d4ea..c7299076 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -136,8 +136,12 @@ pub fn initialize_tracing_log(module_id: &str) -> WorkerGuard { EnvFilter::new("info") } }; - let logging_level = tracing::Level::from_str(&level_env) - .unwrap_or_else(|_| panic!("invalid value for tracing. Got {level_env}")); + let logging_level = if matches!(level_env.as_str(), "info" | "warning" | "error") { + tracing::Level::DEBUG + } else { + tracing::Level::from_str(&level_env) + .unwrap_or_else(|_| panic!("invalid value for tracing. Got {level_env}")) + }; let stdout_log = tracing_subscriber::fmt::layer().pretty(); let (default, guard) = tracing_appender::non_blocking(log_file); let log_file = Layer::new().with_writer(default.with_max_level(logging_level)); diff --git a/examples/builder_log/Cargo.toml b/examples/builder_log/Cargo.toml index 741565f3..4011210e 100644 --- a/examples/builder_log/Cargo.toml +++ b/examples/builder_log/Cargo.toml @@ -6,7 +6,6 @@ rust-version.workspace = true [dependencies] commit-boost = { path = "../../bin" } -cb-common = {path="../../crates/common"} # networking reqwest.workspace = true diff --git a/examples/builder_log/src/main.rs b/examples/builder_log/src/main.rs index b58db7c2..b29dd966 100644 --- a/examples/builder_log/src/main.rs +++ b/examples/builder_log/src/main.rs @@ -1,5 +1,4 @@ use async_trait::async_trait; -use cb_common::module_names::BUILDER_LOG_NAME; use commit_boost::prelude::*; use tracing::{error, info}; @@ -18,7 +17,7 @@ async fn main() { match load_builder_module_config::<()>() { Ok(config) => { info!(module_id = config.id, "Starting module"); - let _guard = initialize_tracing_log(BUILDER_LOG_NAME); + let _guard = initialize_tracing_log(&config.id); let client = BuilderEventClient::new(config.server_port, LogProcessor); if let Err(err) = client.run().await { diff --git a/examples/custom_boost.rs b/examples/custom_boost.rs index 3c408b35..3b5b504f 100644 --- a/examples/custom_boost.rs +++ b/examples/custom_boost.rs @@ -58,9 +58,10 @@ async fn handle_stats(State(state): State>) -> Respo async fn main() { color_eyre::install()?; - let _guard = initialize_tracing_log(CUSTOM_BOOST_NAME); + let (chain, config) = load_pbs_config(); + let _guard = initialize_tracing_log(config.id); info!("Starting custom pbs module"); From 32c4d13c5710f67410eb2183f5f4d16b2f7e050e Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:55:33 +0100 Subject: [PATCH 20/22] stuff --- crates/common/src/config/constants.rs | 2 +- crates/common/src/config/log.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/common/src/config/constants.rs b/crates/common/src/config/constants.rs index ade25fa8..a5469fa0 100644 --- a/crates/common/src/config/constants.rs +++ b/crates/common/src/config/constants.rs @@ -4,7 +4,7 @@ pub const METRICS_SERVER_ENV: &str = "METRICS_SERVER"; pub const SIGNER_SERVER_ENV: &str = "SIGNER_SERVER"; pub const BUILDER_SERVER_ENV: &str = "BUILDER_SERVER"; -pub const CB_BASE_LOG_PATH: &str = "/var/logs/"; +pub const CB_BASE_LOG_PATH: &str = "/var/logs/pbs"; pub const CB_CONFIG_ENV: &str = "CB_CONFIG"; pub const CB_CONFIG_NAME: &str = "/cb-config.toml"; diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs index 2cc3f860..3bab57f8 100644 --- a/crates/common/src/config/log.rs +++ b/crates/common/src/config/log.rs @@ -19,7 +19,7 @@ impl Default for LogsSettings { fn default() -> Self { Self { duration: RollingDuration::Hourly, - host_path: "/var/logs".into(), + host_path: "/var/log/pbs".into(), rust_log: "info".to_string(), } } From 6939733d854ccb1a4c0375cb0d5fd8c180211356 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:56:30 +0100 Subject: [PATCH 21/22] change duration --- config.example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.example.toml b/config.example.toml index 53c92138..ac054924 100644 --- a/config.example.toml +++ b/config.example.toml @@ -41,6 +41,6 @@ type = "events" docker_image = "test_builder_log" [logs] -duration = "hourly" +duration = "daily" host-path="./logs" rust-log="info" \ No newline at end of file From 4bf6cb95880e76014f8ea013ba1680424b74c2cc Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 5 Aug 2024 15:57:45 +0100 Subject: [PATCH 22/22] change CB_BASE_LOG_PATH --- crates/common/src/config/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/common/src/config/constants.rs b/crates/common/src/config/constants.rs index a5469fa0..f5c565aa 100644 --- a/crates/common/src/config/constants.rs +++ b/crates/common/src/config/constants.rs @@ -4,7 +4,7 @@ pub const METRICS_SERVER_ENV: &str = "METRICS_SERVER"; pub const SIGNER_SERVER_ENV: &str = "SIGNER_SERVER"; pub const BUILDER_SERVER_ENV: &str = "BUILDER_SERVER"; -pub const CB_BASE_LOG_PATH: &str = "/var/logs/pbs"; +pub const CB_BASE_LOG_PATH: &str = "/var/logs/commit-boost"; pub const CB_CONFIG_ENV: &str = "CB_CONFIG"; pub const CB_CONFIG_NAME: &str = "/cb-config.toml";