diff --git a/config.example.toml b/config.example.toml index d067c466..6617aa92 100644 --- a/config.example.toml +++ b/config.example.toml @@ -118,6 +118,7 @@ docker_image = "test_da_commit" sleep_secs = 5 # Configuration for how metrics should be collected and scraped +# OPTIONAL, skip metrics collection if missing [metrics] # Path to a `prometheus.yml` file to use in Prometheus. If using a custom config file, be sure to add a # file discovery section as follows: @@ -131,13 +132,13 @@ prometheus_config = "./docker/prometheus.yml" # Whether to start Grafana with built-in dashboards # OPTIONAL, DEFAULT: true use_grafana = true +# Whether to start cadvisor for system monitoring +# OPTIONAL, DEFAULT: true +use_cadvisor = true # Configuration for how logs should be collected and stored -# OPTIONAL +# OPTIONAL, info to stdout if missing [logs] -# Log rotation policy. Supported values: hourly, daily, never -# OPTIONAL, DEFAULT: daily -rotation = "daily" # Path to the log directory # OPTIONAL, DEFAULT: /var/logs/commit-boost log_dir_path = "./logs" diff --git a/configs/minimal.toml b/configs/minimal.toml new file mode 100644 index 00000000..74217f9c --- /dev/null +++ b/configs/minimal.toml @@ -0,0 +1,10 @@ +# Minimal PBS config, with no metrics and only stdout logs + +chain = "Holesky" + +[pbs] +port = 18550 + +[[relays]] +id = "example-relay" +url = "http://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@abc.xyz" diff --git a/configs/pbs_metrics.toml b/configs/pbs_metrics.toml new file mode 100644 index 00000000..45683f3a --- /dev/null +++ b/configs/pbs_metrics.toml @@ -0,0 +1,18 @@ +# PBS + metrics + logs to file + +chain = "Holesky" + +[pbs] +port = 18550 + +[[relays]] +id = "example-relay" +url = "http://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@abc.xyz" + +[metrics] +prometheus_config = "./docker/prometheus.yml" +use_grafana = true +use_cadvisor = false + +[logs] +log_dir_path = "./logs" diff --git a/crates/cli/src/docker_cmd.rs b/crates/cli/src/docker_cmd.rs index 8fab4e2c..b15b2b4f 100644 --- a/crates/cli/src/docker_cmd.rs +++ b/crates/cli/src/docker_cmd.rs @@ -72,13 +72,14 @@ fn is_command_available(command: &str) -> bool { .map_or(false, |output| output.status.success()) } -pub fn handle_docker_start(compose_path: String, env_path: String) -> Result<()> { +pub fn handle_docker_start(compose_path: String, env_path: Option) -> Result<()> { println!("Starting Commit-Boost with compose file: {}", compose_path); - // load env file - let env_file = dotenvy::from_filename_override(env_path)?; - - println!("Loaded env file: {:?}", env_file); + // load env file if present + if let Some(env_path) = env_path { + let env_file = dotenvy::from_filename_override(env_path)?; + println!("Loaded env file: {:?}", env_file); + } // start docker compose run_docker_compose!(compose_path, "up", "-d"); diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 6ea0f82c..25859389 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -1,18 +1,15 @@ -use std::{ - path::{Path, PathBuf}, - vec, -}; +use std::{path::Path, vec}; use cb_common::{ config::{ - CommitBoostConfig, ModuleKind, BUILDER_SERVER_ENV, CB_BASE_LOG_PATH, CB_CONFIG_ENV, - CB_CONFIG_NAME, JWTS_ENV, METRICS_SERVER_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, - PBS_MODULE_NAME, SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS, - SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, SIGNER_KEYS_ENV, SIGNER_MODULE_NAME, - SIGNER_SERVER_ENV, + CommitBoostConfig, LogsSettings, ModuleKind, BUILDER_SERVER_ENV, CB_BASE_LOG_PATH, + CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV, MAX_LOG_FILES_ENV, METRICS_SERVER_ENV, + MODULE_ID_ENV, MODULE_JWT_ENV, PBS_MODULE_NAME, RUST_LOG_ENV, SIGNER_DIR_KEYS, + SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS, SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, + SIGNER_KEYS_ENV, SIGNER_MODULE_NAME, SIGNER_SERVER_ENV, USE_FILE_LOGS_ENV, }, loader::SignerLoader, - utils::{random_jwt, MAX_LOG_FILES_ENV, ROLLING_DURATION_ENV, RUST_LOG_ENV}, + utils::random_jwt, }; use docker_compose_types::{ Compose, ComposeVolume, DependsOnOptions, Environment, Labels, LoggingParameters, MapOrEmpty, @@ -35,12 +32,28 @@ const SIGNER_NETWORK: &str = "signer_network"; /// Builds the docker compose file for the Commit-Boost services // TODO: do more validation for paths, images, etc -#[allow(unused_assignments)] + pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> { println!("Initializing Commit-Boost with config file: {}", config_path); let cb_config = CommitBoostConfig::from_file(&config_path)?; + let metrics_enabled = cb_config.metrics.is_some(); + + // Logging + let logging_envs = if let Some(log_config) = &cb_config.logs { + let mut envs = vec![ + get_env_bool(USE_FILE_LOGS_ENV, true), + get_env_val(RUST_LOG_ENV, &log_config.log_level), + ]; + if let Some(max_files) = log_config.max_log_files { + envs.push(get_env_uval(MAX_LOG_FILES_ENV, max_files as u64)) + } + envs + } else { + vec![] + }; + let mut services = IndexMap::new(); let mut volumes = IndexMap::new(); @@ -49,7 +62,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let mut jwts = IndexMap::new(); // envs to write in .env file - let mut envs = IndexMap::from([(CB_CONFIG_ENV.into(), CB_CONFIG_NAME.into())]); + let mut envs = IndexMap::new(); // targets to pass to prometheus let mut targets = Vec::new(); let metrics_port = 10000; @@ -64,23 +77,6 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let mut exposed_ports_warn = Vec::new(); - // setup pbs service - targets.push(PrometheusTargetConfig { - targets: vec![format!("cb_pbs:{metrics_port}")], - labels: PrometheusLabelsConfig { job: "pbs".to_owned() }, - }); - - let mut pbs_envs = IndexMap::from([ - get_env_same(CB_CONFIG_ENV), - get_env_uval(METRICS_SERVER_ENV, metrics_port as u64), - get_env_val(ROLLING_DURATION_ENV, &cb_config.logs.rotation.to_string()), - get_env_val(RUST_LOG_ENV, &cb_config.logs.log_level), - ]); - if let Some(max_files) = cb_config.logs.max_log_files { - let (key, val) = get_env_uval(MAX_LOG_FILES_ENV, max_files as u64); - pbs_envs.insert(key, val); - } - let mut needs_signer_module = cb_config.pbs.with_signer; // setup modules @@ -89,10 +85,12 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> // TODO: support modules volumes and network let module_cid = format!("cb_{}", module.id.to_lowercase()); - targets.push(PrometheusTargetConfig { - targets: vec![format!("{module_cid}:{metrics_port}")], - labels: PrometheusLabelsConfig { job: module_cid.clone() }, - }); + if metrics_enabled { + targets.push(PrometheusTargetConfig { + targets: vec![format!("{module_cid}:{metrics_port}")], + labels: PrometheusLabelsConfig { job: module_cid.clone() }, + }); + } let module_service = match module.kind { // a commit module needs a JWT and access to the signer network @@ -105,31 +103,35 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> // module ids are assumed unique, so envs dont override each other let mut module_envs = IndexMap::from([ get_env_val(MODULE_ID_ENV, &module.id), - get_env_same(CB_CONFIG_ENV), + get_env_val(CB_CONFIG_ENV, CB_CONFIG_NAME), get_env_interp(MODULE_JWT_ENV, &jwt_name), - get_env_uval(METRICS_SERVER_ENV, metrics_port as u64), get_env_val(SIGNER_SERVER_ENV, &signer_server), - get_env_val(ROLLING_DURATION_ENV, &cb_config.logs.rotation.to_string()), - get_env_val(RUST_LOG_ENV, &cb_config.logs.log_level), ]); - if let Some(max_files) = cb_config.logs.max_log_files { - let (key, val) = get_env_uval(MAX_LOG_FILES_ENV, max_files as u64); + if metrics_enabled { + let (key, val) = get_env_uval(METRICS_SERVER_ENV, metrics_port as u64); module_envs.insert(key, val); } + module_envs.extend(logging_envs.clone()); envs.insert(jwt_name.clone(), jwt.clone()); jwts.insert(module.id.clone(), jwt); - let log_volume = - get_log_volume(cb_config.logs.log_dir_path.clone(), &module.id); + + // networks + let mut module_networks = vec![SIGNER_NETWORK.to_owned()]; + if metrics_enabled { + module_networks.push(METRICS_NETWORK.to_owned()); + } + + // volumes + let mut module_volumes = vec![config_volume.clone()]; + module_volumes.extend(get_log_volume(&cb_config.logs, &module.id)); + Service { container_name: Some(module_cid.clone()), image: Some(module.docker_image), // TODO: allow service to open ports here - networks: Networks::Simple(vec![ - METRICS_NETWORK.to_owned(), - SIGNER_NETWORK.to_owned(), - ]), - volumes: vec![config_volume.clone(), log_volume], + networks: Networks::Simple(module_networks), + volumes: module_volumes, environment: Environment::KvPair(module_envs), depends_on: DependsOnOptions::Simple(vec!["cb_signer".to_owned()]), ..Service::default() @@ -137,28 +139,37 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> } // an event module just needs a port to listen on ModuleKind::Events => { + builder_events_modules.push(format!("{module_cid}:{builder_events_port}")); + // module ids are assumed unique, so envs dont override each other let mut module_envs = IndexMap::from([ get_env_val(MODULE_ID_ENV, &module.id), - get_env_same(CB_CONFIG_ENV), - get_env_uval(METRICS_SERVER_ENV, metrics_port as u64), + get_env_val(CB_CONFIG_ENV, CB_CONFIG_NAME), get_env_val(BUILDER_SERVER_ENV, &builder_events_port.to_string()), - get_env_val(ROLLING_DURATION_ENV, &cb_config.logs.rotation.to_string()), - get_env_val(RUST_LOG_ENV, &cb_config.logs.log_level), ]); - if let Some(max_files) = cb_config.logs.max_log_files { - let (key, val) = get_env_uval(MAX_LOG_FILES_ENV, max_files as u64); + module_envs.extend(logging_envs.clone()); + + if metrics_enabled { + let (key, val) = get_env_uval(METRICS_SERVER_ENV, metrics_port as u64); module_envs.insert(key, val); } - builder_events_modules.push(format!("{module_cid}:{builder_events_port}")); - let log_volume = - get_log_volume(cb_config.logs.log_dir_path.clone(), &module.id); + // networks + let modules_networks = if metrics_enabled { + Networks::Simple(vec![METRICS_NETWORK.to_owned()]) + } else { + Networks::default() + }; + + // volumes + let mut module_volumes = vec![config_volume.clone()]; + module_volumes.extend(get_log_volume(&cb_config.logs, &module.id)); + Service { container_name: Some(module_cid.clone()), image: Some(module.docker_image), - networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - volumes: vec![config_volume.clone(), log_volume], + networks: modules_networks, + volumes: module_volumes, environment: Environment::KvPair(module_envs), depends_on: DependsOnOptions::Simple(vec!["cb_pbs".to_owned()]), ..Service::default() @@ -170,15 +181,41 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> } }; + // setup pbs service + if metrics_enabled { + targets.push(PrometheusTargetConfig { + targets: vec![format!("cb_pbs:{metrics_port}")], + labels: PrometheusLabelsConfig { job: "pbs".to_owned() }, + }); + } + + let mut pbs_envs = IndexMap::from([get_env_val(CB_CONFIG_ENV, CB_CONFIG_NAME)]); + pbs_envs.extend(logging_envs.clone()); + if metrics_enabled { + let (key, val) = get_env_uval(METRICS_SERVER_ENV, metrics_port as u64); + pbs_envs.insert(key, val); + } + if !builder_events_modules.is_empty() { let env = builder_events_modules.join(","); let (k, v) = get_env_val(BUILDER_SERVER_ENV, &env); pbs_envs.insert(k, v); } - let log_volume = get_log_volume(cb_config.logs.log_dir_path.clone(), PBS_MODULE_NAME); + // volumes + let mut pbs_volumes = vec![config_volume.clone()]; + pbs_volumes.extend(get_log_volume(&cb_config.logs, PBS_MODULE_NAME)); + + // networks + let pbs_networs = if metrics_enabled { + Networks::Simple(vec![METRICS_NETWORK.to_owned()]) + } else { + Networks::default() + }; + exposed_ports_warn .push(format!("pbs has an exported port on {}", cb_config.pbs.pbs_config.port)); + let pbs_service = Service { container_name: Some("cb_pbs".to_owned()), image: Some(cb_config.pbs.docker_image), @@ -186,42 +223,42 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> "{}:{}", cb_config.pbs.pbs_config.port, cb_config.pbs.pbs_config.port )]), - networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - volumes: vec![config_volume.clone(), log_volume], + networks: pbs_networs, + volumes: pbs_volumes, environment: Environment::KvPair(pbs_envs), ..Service::default() }; services.insert("cb_pbs".to_owned(), Some(pbs_service)); - // TODO: validate if we have signer modules but not signer config - // setup signer service - if let Some(signer_config) = cb_config.signer { if needs_signer_module { - let log_volume = - get_log_volume(cb_config.logs.log_dir_path.clone(), SIGNER_MODULE_NAME); - let mut volumes = vec![config_volume.clone(), log_volume]; - - targets.push(PrometheusTargetConfig { - targets: vec![format!("cb_signer:{metrics_port}")], - labels: PrometheusLabelsConfig { job: "signer".into() }, - }); + if metrics_enabled { + targets.push(PrometheusTargetConfig { + targets: vec![format!("cb_signer:{metrics_port}")], + labels: PrometheusLabelsConfig { job: "signer".into() }, + }); + } let mut signer_envs = IndexMap::from([ - get_env_same(CB_CONFIG_ENV), + get_env_val(CB_CONFIG_ENV, CB_CONFIG_NAME), get_env_same(JWTS_ENV), - get_env_uval(METRICS_SERVER_ENV, metrics_port as u64), get_env_uval(SIGNER_SERVER_ENV, signer_port as u64), - get_env_val(ROLLING_DURATION_ENV, &cb_config.logs.rotation.to_string()), - get_env_val(RUST_LOG_ENV, &cb_config.logs.log_level), ]); - if let Some(max_files) = cb_config.logs.max_log_files { - let (key, val) = get_env_uval(MAX_LOG_FILES_ENV, max_files as u64); + signer_envs.extend(logging_envs); + if metrics_enabled { + let (key, val) = get_env_uval(METRICS_SERVER_ENV, metrics_port as u64); signer_envs.insert(key, val); } + // write jwts to env + let jwts_json = serde_json::to_string(&jwts).unwrap().clone(); + envs.insert(JWTS_ENV.into(), format!("{jwts_json:?}")); + + // volumes + let mut volumes = vec![config_volume.clone()]; + // TODO: generalize this, different loaders may not need volumes but eg ports match signer_config.loader { SignerLoader::File { key_path } => { @@ -243,17 +280,18 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> } }; - // write jwts to env - let jwts_json = serde_json::to_string(&jwts).unwrap().clone(); - envs.insert(JWTS_ENV.into(), format!("{jwts_json:?}")); + volumes.extend(get_log_volume(&cb_config.logs, SIGNER_MODULE_NAME)); + + // networks + let mut signer_networks = vec![SIGNER_NETWORK.to_owned()]; + if metrics_enabled { + signer_networks.push(METRICS_NETWORK.to_owned()); + } let signer_service = Service { container_name: Some("cb_signer".to_owned()), image: Some(signer_config.docker_image), - networks: Networks::Simple(vec![ - METRICS_NETWORK.to_owned(), - SIGNER_NETWORK.to_owned(), - ]), + networks: Networks::Simple(signer_networks), volumes, environment: Environment::KvPair(signer_envs), ..Service::default() @@ -270,113 +308,129 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let mut compose = Compose::default(); - compose.networks.0.insert( - METRICS_NETWORK.to_owned(), - MapOrEmpty::Map(NetworkSettings { - driver: Some("bridge".to_owned()), - ..NetworkSettings::default() - }), - ); - compose.networks.0.insert( - SIGNER_NETWORK.to_owned(), - MapOrEmpty::Map(NetworkSettings { - driver: Some("bridge".to_owned()), - ..NetworkSettings::default() - }), - ); - - let prom_volume = Volumes::Simple(format!( - "{}:/etc/prometheus/prometheus.yml", - cb_config.metrics.prometheus_config - )); - // TODO: fix path to targets file - let targets_volume = - Volumes::Simple(format!("./{}:/etc/prometheus/targets.json", CB_TARGETS_FILE)); - - let data_volume = Volumes::Simple(format!("{}:/prometheus", PROMETHEUS_DATA_VOLUME)); - - let grafana_data_volume = Volumes::Simple(format!("{}:/var/lib/grafana", GRAFANA_DATA_VOLUME)); - - volumes.insert( - PROMETHEUS_DATA_VOLUME.to_owned(), - MapOrEmpty::Map(ComposeVolume { - driver: Some("local".to_owned()), - driver_opts: IndexMap::default(), - external: None, - labels: Labels::default(), - name: None, - }), - ); - - volumes.insert( - GRAFANA_DATA_VOLUME.to_owned(), - MapOrEmpty::Map(ComposeVolume { - driver: Some("local".to_owned()), - driver_opts: IndexMap::default(), - external: None, - labels: Labels::default(), - name: None, - }), - ); - exposed_ports_warn.push("prometheus has an exported port on 9090".to_string()); - let prometheus_service = Service { - container_name: Some("cb_prometheus".to_owned()), - image: Some("prom/prometheus:latest".to_owned()), - volumes: vec![prom_volume, targets_volume, data_volume], - // to inspect prometheus from localhost - ports: Ports::Short(vec!["9090:9090".to_owned()]), - networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - ..Service::default() - }; + if metrics_enabled { + compose.networks.0.insert( + METRICS_NETWORK.to_owned(), + MapOrEmpty::Map(NetworkSettings { + driver: Some("bridge".to_owned()), + ..NetworkSettings::default() + }), + ); + } + + if needs_signer_module { + compose.networks.0.insert( + SIGNER_NETWORK.to_owned(), + MapOrEmpty::Map(NetworkSettings { + driver: Some("bridge".to_owned()), + ..NetworkSettings::default() + }), + ); + } + + if let Some(metrics_config) = cb_config.metrics { + // prometheus + exposed_ports_warn.push("prometheus has an exported port on 9090".to_string()); - services.insert("cb_prometheus".to_owned(), Some(prometheus_service)); + let prom_volume = Volumes::Simple(format!( + "{}:/etc/prometheus/prometheus.yml", + metrics_config.prometheus_config + )); - if cb_config.metrics.use_grafana { - exposed_ports_warn.push("grafana has an exported port on 3000".to_string()); - let grafana_service = Service { - container_name: Some("cb_grafana".to_owned()), - image: Some("grafana/grafana:latest".to_owned()), - ports: Ports::Short(vec!["3000:3000".to_owned()]), + // TODO: fix path to targets file + let targets_volume = + Volumes::Simple(format!("./{}:/etc/prometheus/targets.json", CB_TARGETS_FILE)); + + let data_volume = Volumes::Simple(format!("{}:/prometheus", PROMETHEUS_DATA_VOLUME)); + + let prometheus_service = Service { + container_name: Some("cb_prometheus".to_owned()), + image: Some("prom/prometheus:latest".to_owned()), + volumes: vec![prom_volume, targets_volume, data_volume], + // to inspect prometheus from localhost + ports: Ports::Short(vec!["9090:9090".to_owned()]), networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - depends_on: DependsOnOptions::Simple(vec!["cb_prometheus".to_owned()]), - environment: Environment::List(vec!["GF_SECURITY_ADMIN_PASSWORD=admin".to_owned()]), - volumes: vec![ - Volumes::Simple( - "./grafana/dashboards:/etc/grafana/provisioning/dashboards".to_owned(), - ), - Volumes::Simple( - "./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned(), - ), - grafana_data_volume, - ], - // TODO: re-enable logging here once we move away from docker logs - logging: Some(LoggingParameters { driver: Some("none".to_owned()), options: None }), ..Service::default() }; - services.insert("cb_grafana".to_owned(), Some(grafana_service)); - } - exposed_ports_warn.push("cadvisor has an exported port on 8080".to_string()); - services.insert( - "cb_cadvisor".to_owned(), - Some(Service { - container_name: Some("cb_cadvisor".to_owned()), - image: Some("gcr.io/cadvisor/cadvisor".to_owned()), - ports: Ports::Short(vec![format!("{cadvisor_port}:8080")]), - networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - volumes: vec![ - Volumes::Simple("/var/run/docker.sock:/var/run/docker.sock:ro".to_owned()), - Volumes::Simple("/sys:/sys:ro".to_owned()), - Volumes::Simple("/var/lib/docker/:/var/lib/docker:ro".to_owned()), - ], - ..Service::default() - }), - ); + services.insert("cb_prometheus".to_owned(), Some(prometheus_service)); + volumes.insert( + PROMETHEUS_DATA_VOLUME.to_owned(), + MapOrEmpty::Map(ComposeVolume { + driver: Some("local".to_owned()), + driver_opts: IndexMap::default(), + external: None, + labels: Labels::default(), + name: None, + }), + ); + + // grafana + if metrics_config.use_grafana { + exposed_ports_warn.push("grafana has an exported port on 3000".to_string()); + + let grafana_data_volume = + Volumes::Simple(format!("{}:/var/lib/grafana", GRAFANA_DATA_VOLUME)); + + let grafana_service = Service { + container_name: Some("cb_grafana".to_owned()), + image: Some("grafana/grafana:latest".to_owned()), + ports: Ports::Short(vec!["3000:3000".to_owned()]), + networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), + depends_on: DependsOnOptions::Simple(vec!["cb_prometheus".to_owned()]), + environment: Environment::List(vec!["GF_SECURITY_ADMIN_PASSWORD=admin".to_owned()]), + volumes: vec![ + Volumes::Simple( + "./grafana/dashboards:/etc/grafana/provisioning/dashboards".to_owned(), + ), + Volumes::Simple( + "./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned(), + ), + grafana_data_volume, + ], + // TODO: re-enable logging here once we move away from docker logs + logging: Some(LoggingParameters { driver: Some("none".to_owned()), options: None }), + ..Service::default() + }; - targets.push(PrometheusTargetConfig { - targets: vec![format!("cb_cadvisor:{cadvisor_port}")], - labels: PrometheusLabelsConfig { job: "cadvisor".to_owned() }, - }); + services.insert("cb_grafana".to_owned(), Some(grafana_service)); + volumes.insert( + GRAFANA_DATA_VOLUME.to_owned(), + MapOrEmpty::Map(ComposeVolume { + driver: Some("local".to_owned()), + driver_opts: IndexMap::default(), + external: None, + labels: Labels::default(), + name: None, + }), + ); + } + + // cadvisor + if metrics_config.use_cadvisor { + exposed_ports_warn.push("cadvisor has an exported port on 8080".to_string()); + + services.insert( + "cb_cadvisor".to_owned(), + Some(Service { + container_name: Some("cb_cadvisor".to_owned()), + image: Some("gcr.io/cadvisor/cadvisor".to_owned()), + ports: Ports::Short(vec![format!("{cadvisor_port}:8080")]), + networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), + volumes: vec![ + Volumes::Simple("/var/run/docker.sock:/var/run/docker.sock:ro".to_owned()), + Volumes::Simple("/sys:/sys:ro".to_owned()), + Volumes::Simple("/var/lib/docker/:/var/lib/docker:ro".to_owned()), + ], + ..Service::default() + }), + ); + targets.push(PrometheusTargetConfig { + targets: vec![format!("cb_cadvisor:{cadvisor_port}")], + labels: PrometheusLabelsConfig { job: "cadvisor".to_owned() }, + }); + } + } compose.services = Services(services); compose.volumes = TopLevelVolumes(volumes); @@ -395,27 +449,36 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> } println!("Compose file written to: {:?}", compose_path); - // write envs to .env file - let envs_str = { - let mut envs_str = String::new(); - for (k, v) in envs { - envs_str.push_str(&format!("{}={}\n", k, v)); - } - envs_str - }; - let env_path = Path::new(&output_dir).join(CB_ENV_FILE); - // TODO: check if file exists already and avoid overwriting - std::fs::write(&env_path, envs_str)?; - println!("Env file written to: {:?}", env_path); - // write prometheus targets to file - let targets_str = serde_json::to_string_pretty(&targets)?; - let targets_path = Path::new(&output_dir).join(CB_TARGETS_FILE); - // TODO: check if file exists already and avoid overwriting - std::fs::write(&targets_path, targets_str)?; - println!("Targets file written to: {:?}", targets_path); + if !targets.is_empty() { + let targets_str = serde_json::to_string_pretty(&targets)?; + let targets_path = Path::new(&output_dir).join(CB_TARGETS_FILE); + // TODO: check if file exists already and avoid overwriting + std::fs::write(&targets_path, targets_str)?; + println!("Targets file written to: {:?}", targets_path); + } - println!("Run with:\n\t`commit-boost start --docker {:?} --env {:?}`", compose_path, env_path); + if envs.is_empty() { + println!("Run with:\n\t`commit-boost start --docker {:?}`", compose_path); + } else { + // write envs to .env file + let envs_str = { + let mut envs_str = String::new(); + for (k, v) in envs { + envs_str.push_str(&format!("{}={}\n", k, v)); + } + envs_str + }; + let env_path = Path::new(&output_dir).join(CB_ENV_FILE); + // TODO: check if file exists already and avoid overwriting + std::fs::write(&env_path, envs_str)?; + println!("Env file written to: {:?}", env_path); + + println!( + "Run with:\n\t`commit-boost start --docker {:?} --env {:?}`", + compose_path, env_path + ); + } Ok(()) } @@ -439,6 +502,10 @@ fn get_env_uval(k: &str, v: u64) -> (String, Option) { (k.into(), Some(SingleValue::Unsigned(v))) } +fn get_env_bool(k: &str, v: bool) -> (String, Option) { + (k.into(), Some(SingleValue::Bool(v))) +} + /// A prometheus target, use to dynamically add targets to the prometheus config #[derive(Debug, Serialize)] struct PrometheusTargetConfig { @@ -451,11 +518,13 @@ struct PrometheusLabelsConfig { job: String, } -fn get_log_volume(host_path: PathBuf, module_id: &str) -> Volumes { - let p = host_path.join(module_id); - Volumes::Simple(format!( - "{}:{}", - p.to_str().expect("could not convert pathbuf to str"), - CB_BASE_LOG_PATH - )) +fn get_log_volume(maybe_config: &Option, module_id: &str) -> Option { + maybe_config.as_ref().map(|config| { + let p = config.log_dir_path.join(module_id); + Volumes::Simple(format!( + "{}:{}", + p.to_str().expect("could not convert pathbuf to str"), + CB_BASE_LOG_PATH + )) + }) } diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 9a6775e4..7f628dfe 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -34,8 +34,8 @@ pub enum Command { compose_path: String, /// Path env file - #[arg(short, long("env"), default_value = CB_ENV_FILE)] - env_path: String, + #[arg(short, long("env"))] + env_path: Option, }, Stop { diff --git a/crates/common/src/config/constants.rs b/crates/common/src/config/constants.rs index 8cbf136d..866b559b 100644 --- a/crates/common/src/config/constants.rs +++ b/crates/common/src/config/constants.rs @@ -3,6 +3,9 @@ pub const MODULE_JWT_ENV: &str = "CB_SIGNER_JWT"; pub const METRICS_SERVER_ENV: &str = "METRICS_SERVER"; pub const SIGNER_SERVER_ENV: &str = "SIGNER_SERVER"; pub const BUILDER_SERVER_ENV: &str = "BUILDER_SERVER"; +pub const MAX_LOG_FILES_ENV: &str = "MAX_LOG_FILES"; +pub const RUST_LOG_ENV: &str = "RUST_LOG"; +pub const USE_FILE_LOGS_ENV: &str = "USE_FILE_LOGS"; pub const CB_BASE_LOG_PATH: &str = "/var/logs/commit-boost"; diff --git a/crates/common/src/config/log.rs b/crates/common/src/config/log.rs index 3e48a24d..774102ef 100644 --- a/crates/common/src/config/log.rs +++ b/crates/common/src/config/log.rs @@ -1,7 +1,4 @@ -use std::{ - fmt::{Display, Formatter}, - path::PathBuf, -}; +use std::path::PathBuf; use serde::{Deserialize, Serialize}; @@ -9,8 +6,6 @@ use super::CB_BASE_LOG_PATH; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct LogsSettings { - #[serde(default)] - pub rotation: RollingDuration, #[serde(default = "default_log_dir_path")] pub log_dir_path: PathBuf, #[serde(default = "default_log_level")] @@ -19,17 +14,6 @@ pub struct LogsSettings { pub max_log_files: Option, } -impl Default for LogsSettings { - fn default() -> Self { - Self { - rotation: RollingDuration::default(), - log_dir_path: default_log_dir_path(), - log_level: default_log_level(), - max_log_files: None, - } - } -} - fn default_log_dir_path() -> PathBuf { CB_BASE_LOG_PATH.into() } @@ -37,22 +21,3 @@ fn default_log_dir_path() -> PathBuf { pub fn default_log_level() -> String { "info".into() } - -#[derive(Clone, Default, Debug, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum RollingDuration { - Hourly, - #[default] - Daily, - Never, -} - -impl Display for RollingDuration { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - RollingDuration::Hourly => write!(f, "hourly"), - RollingDuration::Daily => write!(f, "daily"), - RollingDuration::Never => write!(f, "never"), - } - } -} diff --git a/crates/common/src/config/metrics.rs b/crates/common/src/config/metrics.rs index 6a2ae873..a39ce063 100644 --- a/crates/common/src/config/metrics.rs +++ b/crates/common/src/config/metrics.rs @@ -1,16 +1,19 @@ use eyre::Result; use serde::{Deserialize, Serialize}; -use super::{constants::METRICS_SERVER_ENV, load_env_var}; +use super::{constants::METRICS_SERVER_ENV, load_optional_env_var}; use crate::utils::default_bool; #[derive(Debug, Serialize, Deserialize, Clone)] pub struct MetricsConfig { /// Path to prometheus config file pub prometheus_config: String, - /// Whether to start a grafana service + /// Whether to start the grafana service #[serde(default = "default_bool::")] pub use_grafana: bool, + /// Whether to start the cadvisor service + #[serde(default = "default_bool::")] + pub use_cadvisor: bool, } /// Module runtime config set after init @@ -20,8 +23,11 @@ pub struct ModuleMetricsConfig { } impl ModuleMetricsConfig { - pub fn load_from_env() -> Result { - let server_port = load_env_var(METRICS_SERVER_ENV)?.parse()?; - Ok(ModuleMetricsConfig { server_port }) + pub fn load_from_env() -> Result> { + if let Some(server_port) = load_optional_env_var(METRICS_SERVER_ENV) { + Ok(Some(ModuleMetricsConfig { server_port: server_port.parse()? })) + } else { + Ok(None) + } } } diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 9bd82a14..a5ad60a4 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -28,9 +28,8 @@ pub struct CommitBoostConfig { pub pbs: StaticPbsConfig, pub modules: Option>, pub signer: Option, - pub metrics: MetricsConfig, - #[serde(default)] - pub logs: LogsSettings, + pub metrics: Option, + pub logs: Option, } impl CommitBoostConfig { diff --git a/crates/common/src/config/utils.rs b/crates/common/src/config/utils.rs index cd7bf927..5f1be93f 100644 --- a/crates/common/src/config/utils.rs +++ b/crates/common/src/config/utils.rs @@ -8,6 +8,9 @@ use crate::types::{Jwt, ModuleId}; pub fn load_env_var(env: &str) -> Result { std::env::var(env).wrap_err(format!("{env} is not set")) } +pub fn load_optional_env_var(env: &str) -> Option { + std::env::var(env).ok() +} pub fn load_from_file(path: &str) -> Result { let config_file = diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index 1b727967..24f1a598 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -16,7 +16,10 @@ use tracing_appender::{non_blocking::WorkerGuard, rolling::Rotation}; use tracing_subscriber::{fmt::Layer, prelude::*, EnvFilter}; use crate::{ - config::{default_log_level, RollingDuration, CB_BASE_LOG_PATH, PBS_MODULE_NAME}, + config::{ + default_log_level, load_optional_env_var, CB_BASE_LOG_PATH, MAX_LOG_FILES_ENV, + PBS_MODULE_NAME, RUST_LOG_ENV, USE_FILE_LOGS_ENV, + }, pbs::HEADER_VERSION_VALUE, types::Chain, }; @@ -24,12 +27,6 @@ use crate::{ const SECONDS_PER_SLOT: u64 = 12; const MILLIS_PER_SECOND: u64 = 1_000; -pub const ROLLING_DURATION_ENV: &str = "ROLLING_DURATION"; - -pub const MAX_LOG_FILES_ENV: &str = "MAX_LOG_FILES"; - -pub const RUST_LOG_ENV: &str = "RUST_LOG"; - pub fn timestamp_of_slot_start_millis(slot: u64, chain: Chain) -> u64 { let seconds_since_genesis = chain.genesis_time_sec() + slot * SECONDS_PER_SLOT; seconds_since_genesis * MILLIS_PER_SECOND @@ -127,31 +124,7 @@ pub const fn default_u256() -> U256 { // LOGGING pub fn initialize_tracing_log(module_id: &str) -> WorkerGuard { - // Log all events to a rolling log file. - let mut builder = - tracing_appender::rolling::Builder::new().filename_prefix(module_id.to_lowercase()); - if let Ok(value) = env::var(MAX_LOG_FILES_ENV) { - builder = - builder.max_log_files(value.parse().expect("MAX_LOG_FILES is not a valid usize value")); - } - - let rotation = match env::var(ROLLING_DURATION_ENV) - .unwrap_or(RollingDuration::default().to_string()) - .as_str() - { - "hourly" => Rotation::HOURLY, - "daily" => Rotation::DAILY, - "never" => Rotation::NEVER, - _ => panic!("unknown rotation value"), - }; - - let log_file = builder - .rotation(rotation) - .build(CB_BASE_LOG_PATH) - .expect("failed building rolling file appender"); - let level_env = std::env::var(RUST_LOG_ENV).unwrap_or(default_log_level()); - // Log level for stdout let stdout_log_level = match level_env.parse::() { Ok(f) => f, @@ -160,26 +133,52 @@ pub fn initialize_tracing_log(module_id: &str) -> WorkerGuard { Level::INFO } }; + let stdout_filter = format_crates_filter(Level::INFO.as_str(), stdout_log_level.as_str()); + + let use_file_logs = load_optional_env_var(USE_FILE_LOGS_ENV) + .map(|s| s.parse().expect("failed to parse USE_FILE_LOGS")) + .unwrap_or(false); + + if use_file_logs { + let (writer, guard) = tracing_appender::non_blocking(std::io::stdout()); + let stdout_layer = tracing_subscriber::fmt::layer() + .with_target(false) + .with_writer(writer) + .with_filter(stdout_filter); + tracing_subscriber::registry().with(stdout_layer).init(); + guard + } else { + // Log all events to a rolling log file. + let mut builder = + tracing_appender::rolling::Builder::new().filename_prefix(module_id.to_lowercase()); + if let Ok(value) = env::var(MAX_LOG_FILES_ENV) { + builder = builder + .max_log_files(value.parse().expect("MAX_LOG_FILES is not a valid usize value")); + } + let file_appender = builder + .rotation(Rotation::DAILY) + .build(CB_BASE_LOG_PATH) + .expect("failed building rolling file appender"); - // at least debug for file logs - let file_log_level = stdout_log_level.max(Level::DEBUG); + let (writer, guard) = tracing_appender::non_blocking(file_appender); - let stdout_log_filter = format_crates_filter(Level::INFO.as_str(), stdout_log_level.as_str()); - let file_log_filter = format_crates_filter(Level::INFO.as_str(), file_log_level.as_str()); + // at least debug for file logs + let file_log_level = stdout_log_level.max(Level::DEBUG); + let file_log_filter = format_crates_filter(Level::INFO.as_str(), file_log_level.as_str()); - let stdout_log = - tracing_subscriber::fmt::layer().with_target(false).with_filter(stdout_log_filter); - let (default_writer, guard) = tracing_appender::non_blocking(log_file); - let file_log = Layer::new() - .json() - .with_current_span(false) - .with_span_list(true) - .with_writer(default_writer) - .with_filter(file_log_filter); + let stdout_layer = + tracing_subscriber::fmt::layer().with_target(false).with_filter(stdout_filter); - tracing_subscriber::registry().with(stdout_log.and_then(file_log)).init(); + let file_layer = Layer::new() + .json() + .with_current_span(false) + .with_span_list(true) + .with_writer(writer) + .with_filter(file_log_filter); - guard + tracing_subscriber::registry().with(stdout_layer.and_then(file_layer)).init(); + guard + } } pub fn initialize_pbs_tracing_log() -> WorkerGuard { diff --git a/crates/metrics/src/provider.rs b/crates/metrics/src/provider.rs index 939b2b34..39852a0d 100644 --- a/crates/metrics/src/provider.rs +++ b/crates/metrics/src/provider.rs @@ -10,7 +10,7 @@ use axum::{ use cb_common::config::ModuleMetricsConfig; use prometheus::{Encoder, Registry, TextEncoder}; use tokio::net::TcpListener; -use tracing::{error, info, trace}; +use tracing::{error, info, trace, warn}; pub struct MetricsProvider { config: ModuleMetricsConfig, @@ -22,18 +22,21 @@ impl MetricsProvider { MetricsProvider { config, registry } } - pub fn from_registry(registry: Registry) -> eyre::Result { - let config = ModuleMetricsConfig::load_from_env()?; - Ok(MetricsProvider { config, registry }) + pub fn from_registry(registry: Registry) -> eyre::Result> { + Ok(ModuleMetricsConfig::load_from_env()?.map(|config| MetricsProvider { config, registry })) } pub fn load_and_run(registry: Registry) -> eyre::Result<()> { - let provider = MetricsProvider::from_registry(registry)?; - tokio::spawn(async move { - if let Err(err) = provider.run().await { - error!("Metrics server error: {:?}", err); - } - }); + if let Some(provider) = MetricsProvider::from_registry(registry)? { + tokio::spawn(async move { + if let Err(err) = provider.run().await { + error!("Metrics server error: {:?}", err); + } + }); + } else { + warn!("No metrics server configured"); + } + Ok(()) }