log rotation

Cargo.toml

@@ -54,6 +54,7 @@ serde_yaml = "0.9.33"
 # telemetry
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+tracing-appender = "0.2.3"
 prometheus = "0.13.4"
 
 # crypto

bin/default_pbs.rs

@@ -11,10 +11,9 @@ async fn main() -> Result<()> {
         std::env::set_var("RUST_BACKTRACE", "1");
     }
 
-    initialize_tracing_log();
-
     // TODO: handle errors
     let pbs_config = load_pbs_config().expect("failed to load pbs config");
+    initialize_tracing_log(pbs_config.logs_settings.clone(), "pbs");
     let state = PbsState::<()>::new(pbs_config);
 
     PbsService::init_metrics()?;

bin/signer_module.rs

@@ -11,8 +11,7 @@ async fn main() -> Result<()> {
         std::env::set_var("RUST_BACKTRACE", "1");
     }
 
-    initialize_tracing_log();
-
     let config = StartSignerConfig::load_from_env()?;
+    initialize_tracing_log(config.logs_settings.clone(), "signer");
     SigningService::run(config).await
 }

config.example.toml

@@ -39,3 +39,12 @@ sleep_secs = 5
 id = "BUILDER_LOG"
 type = "events"
 docker_image = "test_builder_log"
+
+[logs]
+duration = "hourly"
+
+[logs.prefixes]
+pbs="pbs"
+signer="signer"
+da_commit="da"
+logs="logs"

crates/cli/src/docker_init.rs

@@ -288,7 +288,14 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()>
             networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]),
             depends_on: DependsOnOptions::Simple(vec!["cb_prometheus".to_owned()]),
             environment: Environment::List(vec!["GF_SECURITY_ADMIN_PASSWORD=admin".to_owned()]),
-            volumes: vec![Volumes::Simple("./grafana/dashboards:/etc/grafana/provisioning/dashboards".to_owned()), Volumes::Simple("./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned())],
+            volumes: vec![
+                Volumes::Simple(
+                    "./grafana/dashboards:/etc/grafana/provisioning/dashboards".to_owned(),
+                ),
+                Volumes::Simple(
+                    "./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned(),
+                ),
+            ],
             // TODO: re-enable logging here once we move away from docker logs
             logging: Some(LoggingParameters { driver: Some("none".to_owned()), options: None }),
             ..Service::default()

crates/common/Cargo.toml

@@ -28,6 +28,7 @@ serde_json.workspace = true
 # telemetry
 tracing.workspace = true
 tracing-subscriber.workspace = true
+tracing-appender.workspace = true
 
 # crypto
 blst.workspace = true

crates/common/src/config/mod.rs

@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
 use eyre::Result;
 use serde::{Deserialize, Serialize};
 
@@ -26,6 +28,8 @@ pub struct CommitBoostConfig {
     pub modules: Option<Vec<StaticModuleConfig>>,
     pub signer: Option<SignerConfig>,
     pub metrics: MetricsConfig,
+    #[serde(default)]
+    pub logs: LogsSettings,
 }
 
 impl CommitBoostConfig {
@@ -37,3 +41,27 @@ impl CommitBoostConfig {
         load_file_from_env(CB_CONFIG_ENV)
     }
 }
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct LogsSettings {
+    pub duration: RollingDuration,
+    pub prefixes: HashMap<String, String>,
+}
+
+impl Default for LogsSettings {
+    fn default() -> Self {
+        Self { duration: RollingDuration::Hourly, prefixes: Default::default() }
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub enum RollingDuration {
+    #[serde(rename = "minutely")]
+    Minutely,
+    #[serde(rename = "hourly")]
+    Hourly,
+    #[serde(rename = "daily")]
+    Daily,
+    #[serde(rename = "never")]
+    Never,
+}

crates/common/src/config/module.rs

@@ -2,6 +2,7 @@ use eyre::{ContextCompat, Result};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use toml::Table;
 
+use super::LogsSettings;
 use crate::{
     commit::client::SignerClient,
     config::{
@@ -44,6 +45,8 @@ pub struct StartCommitModuleConfig<T = ()> {
     pub signer_client: SignerClient,
     /// Opaque module config
     pub extra: T,
+
+    pub logs_settings: LogsSettings,
 }
 
 /// Loads a module config from the environment and config file:
@@ -76,6 +79,7 @@ pub fn load_commit_module_config<T: DeserializeOwned>() -> Result<StartCommitMod
     struct StubConfig<U> {
         chain: Chain,
         modules: Vec<ThisModule<U>>,
+        logs: LogsSettings,
     }
 
     // load module config including the extra data (if any)
@@ -105,6 +109,7 @@ pub fn load_commit_module_config<T: DeserializeOwned>() -> Result<StartCommitMod
         chain: cb_config.chain,
         signer_client,
         extra: module_config.extra,
+        logs_settings: cb_config.logs,
     })
 }
 
@@ -118,6 +123,8 @@ pub struct StartBuilderModuleConfig<T> {
     pub server_port: u16,
     /// Opaque module config
     pub extra: T,
+
+    pub logs_settings: LogsSettings,
 }
 
 pub fn load_builder_module_config<T: DeserializeOwned>() -> eyre::Result<StartBuilderModuleConfig<T>>
@@ -145,6 +152,7 @@ pub fn load_builder_module_config<T: DeserializeOwned>() -> eyre::Result<StartBu
     struct StubConfig<U> {
         chain: Chain,
         modules: Vec<ThisModule<U>>,
+        logs: LogsSettings,
     }
 
     // load module config including the extra data (if any)
@@ -172,5 +180,6 @@ pub fn load_builder_module_config<T: DeserializeOwned>() -> eyre::Result<StartBu
         chain: cb_config.chain,
         server_port: builder_events_port,
         extra: module_config.extra,
+        logs_settings: cb_config.logs,
     })
 }

crates/common/src/config/pbs.rs

@@ -6,7 +6,7 @@ use alloy::primitives::U256;
 use eyre::Result;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 
-use super::{constants::PBS_DEFAULT_IMAGE, CommitBoostConfig};
+use super::{constants::PBS_DEFAULT_IMAGE, CommitBoostConfig, LogsSettings};
 use crate::{
     commit::client::SignerClient,
     config::{load_env_var, load_file_from_env, CB_CONFIG_ENV, MODULE_JWT_ENV, SIGNER_SERVER_ENV},
@@ -90,6 +90,8 @@ pub struct PbsModuleConfig<T = ()> {
     pub event_publiher: Option<BuilderEventPublisher>,
     /// Opaque module config
     pub extra: T,
+
+    pub logs_settings: LogsSettings,
 }
 
 fn default_pbs() -> String {
@@ -110,6 +112,7 @@ pub fn load_pbs_config() -> Result<PbsModuleConfig<()>> {
         signer_client: None,
         event_publiher: maybe_publiher,
         extra: (),
+        logs_settings: config.logs,
     })
 }
 
@@ -128,6 +131,7 @@ pub fn load_pbs_custom_config<T: DeserializeOwned>() -> Result<PbsModuleConfig<T
         chain: Chain,
         relays: Vec<RelayConfig>,
         pbs: CustomPbsConfig<U>,
+        logs: LogsSettings,
     }
 
     // load module config including the extra data (if any)
@@ -152,5 +156,6 @@ pub fn load_pbs_custom_config<T: DeserializeOwned>() -> Result<PbsModuleConfig<T
         signer_client,
         event_publiher: maybe_publiher,
         extra: cb_config.pbs.extra,
+        logs_settings: cb_config.logs,
     })
 }

crates/common/src/config/signer.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 use super::{
     constants::{SIGNER_IMAGE, SIGNER_SERVER_ENV},
     utils::{load_env_var, load_jwts},
-    CommitBoostConfig,
+    CommitBoostConfig, LogsSettings,
 };
 use crate::{loader::SignerLoader, types::Chain};
 
@@ -29,6 +29,7 @@ pub struct StartSignerConfig {
     pub loader: SignerLoader,
     pub server_port: u16,
     pub jwts: HashMap<String, String>,
+    pub logs_settings: LogsSettings,
 }
 
 impl StartSignerConfig {
@@ -43,6 +44,7 @@ impl StartSignerConfig {
             loader: config.signer.expect("Signer config is missing").loader,
             server_port,
             jwts,
+            logs_settings: config.logs,
         })
     }
 }

crates/common/src/utils.rs

@@ -7,9 +7,14 @@ use alloy::{
 use blst::min_pk::{PublicKey, Signature};
 use rand::{distributions::Alphanumeric, Rng};
 use reqwest::header::HeaderMap;
-use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
+use tracing_subscriber::{
+    fmt, fmt::writer::MakeWriterExt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter,
+};
 
-use crate::types::Chain;
+use crate::{
+    config::{LogsSettings, RollingDuration},
+    types::Chain,
+};
 
 const SECONDS_PER_SLOT: u64 = 12;
 const MILLIS_PER_SECOND: u64 = 1_000;
@@ -111,9 +116,21 @@ pub const fn default_u256() -> U256 {
 
 // LOGGING
 // TODO: more customized logging + logging guard
-pub fn initialize_tracing_log() {
+pub fn initialize_tracing_log(logs_settings: LogsSettings, module_name: &str) {
     let level_env = std::env::var("RUST_LOG").unwrap_or("info".to_owned());
-
+    let prefix = logs_settings
+        .prefixes
+        .get(module_name)
+        .unwrap_or_else(|| panic!("prefix not found for module name {module_name}"));
+    // Log all events to a rolling log file.
+    let logfile = match logs_settings.duration {
+        RollingDuration::Minutely => tracing_appender::rolling::minutely("/var/logs", prefix),
+        RollingDuration::Hourly => tracing_appender::rolling::hourly("/var/logs", prefix),
+        RollingDuration::Daily => tracing_appender::rolling::daily("/var/logs", prefix),
+        RollingDuration::Never => tracing_appender::rolling::never("/var/logs", prefix),
+    };
+    // Log `INFO` and above to stdout.
+    let stdout = std::io::stdout.with_max_level(tracing::Level::INFO);
     let filter = match level_env.parse::<EnvFilter>() {
         Ok(f) => f,
         Err(_) => {
@@ -124,7 +141,7 @@ pub fn initialize_tracing_log() {
 
     tracing_subscriber::registry()
         .with(filter)
-        .with(fmt::layer().with_target(false))
+        .with(fmt::layer().with_writer(stdout.and(logfile)))
         .try_init()
         .unwrap();
 }

crates/pbs/src/mev_boost/get_header.rs

@@ -47,7 +47,7 @@ pub async fn get_header<S: BuilderApiState>(
             "late in slot, skipping relay requests"
         );
 
-        return Ok(None)
+        return Ok(None);
     }
 
     let (_, slot_uuid) = state.get_slot_and_uuid();
@@ -168,15 +168,15 @@ async fn send_timed_get_header(
                 .max_by_key(|(start_time, _)| *start_time)
             {
                 debug!(n_headers, "TG: received headers from relay");
-                return Ok(maybe_header)
+                return Ok(maybe_header);
             } else {
                 // all requests failed
                 warn!("TG: no headers received");
 
                 return Err(PbsError::RelayResponse {
                     error_msg: "no headers received".to_string(),
                     code: TIMEOUT_ERROR_CODE,
-                })
+                });
             }
         }
     }
@@ -255,7 +255,7 @@ async fn send_one_get_header(
             response = ?response_bytes,
             "no header from relay"
         );
-        return Ok((start_request_time, None))
+        return Ok((start_request_time, None));
     }
 
     let get_header_response: GetHeaderReponse = serde_json::from_slice(&response_bytes)?;
@@ -293,7 +293,7 @@ fn validate_header(
     let value = signed_header.message.value();
 
     if block_hash == B256::ZERO {
-        return Err(ValidationError::EmptyBlockhash)
+        return Err(ValidationError::EmptyBlockhash);
     }
 
     if parent_hash != signed_header.message.header.parent_hash {
@@ -315,7 +315,7 @@ fn validate_header(
         return Err(ValidationError::PubkeyMismatch {
             expected: expected_relay_pubkey,
             got: received_relay_pubkey,
-        })
+        });
     }
 
     if !skip_sig_verify {

crates/pbs/src/mev_boost/status.rs

@@ -63,7 +63,7 @@ async fn send_relay_check(relay: &RelayClient, headers: HeaderMap) -> Result<(),
             RELAY_STATUS_CODE
                 .with_label_values(&[TIMEOUT_ERROR_CODE_STR, STATUS_ENDPOINT_TAG, &relay.id])
                 .inc();
-            return Err(err.into())
+            return Err(err.into());
         }
     };
     let request_latency = start_request.elapsed();
@@ -82,7 +82,7 @@ async fn send_relay_check(relay: &RelayClient, headers: HeaderMap) -> Result<(),
         };
 
         error!(?err, "status failed");
-        return Err(err)
+        return Err(err);
     };
 
     debug!(?code, latency = ?request_latency, "status passed");

crates/pbs/src/mev_boost/submit_block.rs

@@ -83,7 +83,7 @@ async fn send_submit_block(
                     &relay.id,
                 ])
                 .inc();
-            return Err(err.into())
+            return Err(err.into());
         }
     };
     let request_latency = start_request.elapsed();
@@ -105,7 +105,7 @@ async fn send_submit_block(
 
         // we request payload to all relays, but some may have not received it
         warn!(?err, "failed to get payload (this might be ok if other relays have it)");
-        return Err(err)
+        return Err(err);
     };
 
     let block_response: SubmitBlindedBlockResponse = serde_json::from_slice(&response_bytes)?;
@@ -120,7 +120,7 @@ async fn send_submit_block(
         return Err(PbsError::Validation(ValidationError::BlockHashMismatch {
             expected: signed_blinded_block.block_hash(),
             got: block_response.block_hash(),
-        }))
+        }));
     }
 
     if let Some(blobs) = &block_response.data.blobs_bundle {
@@ -134,7 +134,7 @@ async fn send_submit_block(
                 got_blobs: blobs.blobs.len(),
                 got_commitments: blobs.commitments.len(),
                 got_proofs: blobs.proofs.len(),
-            }))
+            }));
         }
 
         for (i, comm) in expected_committments.iter().enumerate() {
@@ -144,7 +144,7 @@ async fn send_submit_block(
                     expected: format!("{comm}"),
                     got: format!("{}", blobs.commitments[i]),
                     index: i,
-                }))
+                }));
             }
         }
     }