From 1527ff4dea2ecc9750c54d951b99c727929f52fc Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Thu, 14 Mar 2019 10:15:54 +0000 Subject: [PATCH 1/7] graph-builder/cargo: sort dependencies --- graph-builder/Cargo.toml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/graph-builder/Cargo.toml b/graph-builder/Cargo.toml index 54689ed70..f0a65161e 100644 --- a/graph-builder/Cargo.toml +++ b/graph-builder/Cargo.toml @@ -7,11 +7,15 @@ authors = ["Alex Crawford "] actix-web = "^0.7.8" cincinnati = { path = "../cincinnati" } commons = { path = "../commons" } +dkregistry = { git = "https://github.com/camallo/dkregistry-rs.git", rev = "a10c1aba444ce5c07e113c81160eb67c9711e327" } env_logger = "^0.6.0" -itertools = "^0.7.8" failure = "^0.1.1" flate2 = "^1.0.1" +futures = "0.1" +itertools = "^0.7.8" log = "^0.4.3" +quay = { path = "../quay" } +regex = "^1.1.0" reqwest = "^0.9.0" semver = { version = "^0.9.0", features = [ "serde" ] } serde = "^1.0.70" @@ -20,12 +24,7 @@ serde_json = "^1.0.22" structopt = "^0.2.10" tar = "^0.4.16" tokio = "0.1" -dkregistry = { git = "https://github.com/camallo/dkregistry-rs.git", rev = "a10c1aba444ce5c07e113c81160eb67c9711e327" } -futures = "0.1" -quay = { path = "../quay" } url = "^1.7.2" -regex = "^1.1.0" - [features] test-net = [] From 39df2448d325c86fe9986427b5987490443c84dd Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Thu, 14 Mar 2019 10:17:06 +0000 Subject: [PATCH 2/7] graph-builder/cargo: add dependencies for prometheus metrics --- graph-builder/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/graph-builder/Cargo.toml b/graph-builder/Cargo.toml index f0a65161e..8ae3e7467 100644 --- a/graph-builder/Cargo.toml +++ b/graph-builder/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" authors = ["Alex Crawford "] [dependencies] +actix = "^0.7.6" actix-web = "^0.7.8" cincinnati = { path = "../cincinnati" } commons = { path = "../commons" } @@ -13,7 +14,9 @@ failure = "^0.1.1" flate2 = "^1.0.1" futures = "0.1" itertools = "^0.7.8" +lazy_static = "^1.2.0" log = "^0.4.3" +prometheus = "^0.4.2" quay = { path = "../quay" } regex = "^1.1.0" reqwest = "^0.9.0" From 734761150a12da527071e3b171a40a0c0d692675 Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Thu, 14 Mar 2019 10:19:18 +0000 Subject: [PATCH 3/7] cargo: refresh lockfile --- Cargo.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index eec32d5a0..cf75c9f68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -660,6 +660,7 @@ dependencies = [ name = "graph-builder" version = "0.1.0" dependencies = [ + "actix 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", "actix-web 0.7.18 (registry+https://github.com/rust-lang/crates.io-index)", "cincinnati 0.1.0", "commons 0.1.0", @@ -669,7 +670,9 @@ dependencies = [ "flate2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "prometheus 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "quay 0.0.0-dev", "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.8 (registry+https://github.com/rust-lang/crates.io-index)", From f97ce7af70040b701cd9698764079b1b88348201 Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Thu, 14 Mar 2019 10:21:28 +0000 Subject: [PATCH 4/7] graph-builder: sort extern crate statements --- graph-builder/src/lib.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/graph-builder/src/lib.rs b/graph-builder/src/lib.rs index 7a577d3c1..2bc2b706c 100644 --- a/graph-builder/src/lib.rs +++ b/graph-builder/src/lib.rs @@ -1,27 +1,27 @@ +extern crate actix_web; extern crate cincinnati; extern crate commons; extern crate dkregistry; extern crate env_logger; +#[macro_use] +extern crate failure; extern crate flate2; extern crate futures; extern crate itertools; +#[macro_use] +extern crate log; +extern crate quay; +extern crate regex; extern crate reqwest; extern crate semver; extern crate serde; #[macro_use] extern crate serde_derive; -extern crate actix_web; extern crate serde_json; -extern crate tar; -extern crate tokio; -#[macro_use] -extern crate failure; -#[macro_use] -extern crate log; #[macro_use] extern crate structopt; -extern crate quay; -extern crate regex; +extern crate tar; +extern crate tokio; pub mod config; pub mod graph; From 4c1cce28f5d9fdc95da946d549de5f3db9385fbd Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Thu, 14 Mar 2019 10:59:54 +0000 Subject: [PATCH 5/7] graph-builder: add metrics handler --- graph-builder/src/lib.rs | 3 +++ graph-builder/src/metrics.rs | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 graph-builder/src/metrics.rs diff --git a/graph-builder/src/lib.rs b/graph-builder/src/lib.rs index 2bc2b706c..ad046b17f 100644 --- a/graph-builder/src/lib.rs +++ b/graph-builder/src/lib.rs @@ -10,6 +10,8 @@ extern crate futures; extern crate itertools; #[macro_use] extern crate log; +#[macro_use] +extern crate prometheus; extern crate quay; extern crate regex; extern crate reqwest; @@ -25,5 +27,6 @@ extern crate tokio; pub mod config; pub mod graph; +pub mod metrics; pub mod registry; pub mod release; diff --git a/graph-builder/src/metrics.rs b/graph-builder/src/metrics.rs new file mode 100644 index 000000000..7ebb9e023 --- /dev/null +++ b/graph-builder/src/metrics.rs @@ -0,0 +1,21 @@ +//! Metrics service. + +use actix_web::{HttpRequest, HttpResponse}; +use futures::future; +use futures::prelude::*; +use prometheus; + +/// Serve metrics requests (Prometheus textual format). +pub fn serve(_req: HttpRequest<()>) -> Box> { + use prometheus::Encoder; + + let resp = future::ok(prometheus::gather()) + .and_then(|metrics| { + let tenc = prometheus::TextEncoder::new(); + let mut buf = vec![]; + tenc.encode(&metrics, &mut buf).and(Ok(buf)) + }) + .from_err() + .map(|content| HttpResponse::Ok().body(content)); + Box::new(resp) +} From e9e3f998f0398195ac06b440ff085c1e1b5563cc Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Thu, 14 Mar 2019 11:01:00 +0000 Subject: [PATCH 6/7] graph-builder: add a status service for metrics This adds an initial status service to serve the `/metrics` endpoint. It is initially hardcoded to bind to 0.0.0.0:9080 (pending configuration rework). --- graph-builder/src/main.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/graph-builder/src/main.rs b/graph-builder/src/main.rs index ad307cf8b..e5d9e8749 100644 --- a/graph-builder/src/main.rs +++ b/graph-builder/src/main.rs @@ -12,21 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +extern crate actix; extern crate actix_web; extern crate failure; extern crate graph_builder; extern crate log; extern crate structopt; -use graph_builder::{config, graph}; +use graph_builder::{config, graph, metrics}; use actix_web::{http::Method, middleware::Logger, server, App}; use failure::Error; use log::LevelFilter; -use std::thread; +use std::{net, thread}; use structopt::StructOpt; fn main() -> Result<(), Error> { + let sys = actix::System::new("graph-builder"); + let opts = config::Options::from_args(); env_logger::Builder::from_default_env() @@ -50,6 +53,20 @@ fn main() -> Result<(), Error> { thread::spawn(move || graph::run(&opts, &state)); } + // TODO(lucab): make these configurable. + let status_address: net::IpAddr = net::Ipv4Addr::UNSPECIFIED.into(); + let status_port = 9080; + + // Status service. + server::new(|| { + App::new() + .middleware(Logger::default()) + .route("/metrics", Method::GET, metrics::serve) + }) + .bind((status_address, status_port))? + .start(); + + // Main service. server::new(move || { let app_prefix = app_prefix.clone(); let state = state.clone(); @@ -59,6 +76,8 @@ fn main() -> Result<(), Error> { .route("/v1/graph", Method::GET, graph::index) }) .bind(addr)? - .run(); + .start(); + + sys.run(); Ok(()) } From 630d107024b3a07df6ef725613d436010c99b44a Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Thu, 14 Mar 2019 11:02:39 +0000 Subject: [PATCH 7/7] graph-builder: instrument with initial graph metrics This adds some initial instrumentation to record basic metrics about incoming requests, total and failed scrapes, fetched releases, and nodes recorded in the processed DAG. --- graph-builder/src/graph.rs | 47 +++++++++++++++++++++++++++++++++----- graph-builder/src/lib.rs | 2 ++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/graph-builder/src/graph.rs b/graph-builder/src/graph.rs index 7e88e8ab9..a32f1a4c0 100644 --- a/graph-builder/src/graph.rs +++ b/graph-builder/src/graph.rs @@ -17,13 +17,44 @@ use cincinnati::{plugins, AbstractRelease, Graph, Release, CONTENT_TYPE}; use commons::GraphError; use config; use failure::Error; +use prometheus::{Counter, IntGauge}; use registry::{self, Registry}; use serde_json; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, RwLock}; use std::thread; +lazy_static! { + static ref GRAPH_FINAL_RELEASES: IntGauge = register_int_gauge!( + "cincinnati_gb_graph_final_releases", + "Number of releases in the final graph, after processing" + ) + .unwrap(); + static ref GRAPH_UPSTREAM_RAW_RELEASES: IntGauge = register_int_gauge!( + "cincinnati_gb_graph_upstream_raw_releases", + "Number of releases fetched from upstream, before processing" + ) + .unwrap(); + static ref UPSTREAM_SCRAPES: Counter = register_counter!( + "cincinnati_gb_graph_upstream_scrapes_total", + "Total number of upstream scrapes" + ) + .unwrap(); + static ref UPSTREAM_ERRORS: Counter = register_counter!( + "cincinnati_gb_graph_upstream_errors_total", + "Total number of upstream scraping errors" + ) + .unwrap(); + static ref V1_GRAPH_INCOMING_REQS: Counter = register_counter!( + "cincinnati_gb_v1_graph_incoming_requests_total", + "Total number of incoming HTTP client request to /v1/graph" + ) + .unwrap(); +} + pub fn index(req: HttpRequest) -> Result { + V1_GRAPH_INCOMING_REQS.inc(); + // Check that the client can accept JSON media type. commons::ensure_content_type(req.headers(), CONTENT_TYPE)?; @@ -113,14 +144,17 @@ pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { debug!("graph update triggered"); - let releases = match registry::fetch_releases( + let scrape = registry::fetch_releases( ®istry, &opts.repository, username.as_ref().map(String::as_ref), password.as_ref().map(String::as_ref), &mut cache, &opts.quay_manifestref_key, - ) { + ); + UPSTREAM_SCRAPES.inc(); + + let releases = match scrape { Ok(releases) => { if releases.is_empty() { warn!( @@ -132,11 +166,13 @@ pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { releases } Err(err) => { + UPSTREAM_ERRORS.inc(); err.iter_chain() .for_each(|cause| error!("failed to fetch all release metadata: {}", cause)); vec![] } }; + GRAPH_UPSTREAM_RAW_RELEASES.set(releases.len() as i64); let graph = match create_graph(releases) { Ok(graph) => graph, @@ -164,10 +200,9 @@ pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { match serde_json::to_string(&graph) { Ok(json) => { *state.json.write().expect("json lock has been poisoned") = json; - debug!( - "graph update completed, {} valid releases", - graph.releases_count() - ); + let nodes_count = graph.releases_count(); + GRAPH_FINAL_RELEASES.set(nodes_count as i64); + debug!("graph update completed, {} valid releases", nodes_count); } Err(err) => error!("Failed to serialize graph: {}", err), }; diff --git a/graph-builder/src/lib.rs b/graph-builder/src/lib.rs index ad046b17f..16d96ea0b 100644 --- a/graph-builder/src/lib.rs +++ b/graph-builder/src/lib.rs @@ -9,6 +9,8 @@ extern crate flate2; extern crate futures; extern crate itertools; #[macro_use] +extern crate lazy_static; +#[macro_use] extern crate log; #[macro_use] extern crate prometheus;