diff --git a/.github/dependabot.yml b/.github/dependabot.yml index db0413ce..2d450de3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,5 +3,5 @@ updates: - package-ecosystem: cargo directory: "/" schedule: - interval: daily - open-pull-requests-limit: 10 \ No newline at end of file + interval: weekly + open-pull-requests-limit: 10 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92882d06..2c5dd6a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,88 +10,73 @@ jobs: fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install Rust Nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - override: true - components: rustfmt, clippy + run: rustup default nightly - name: Check Formatting - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check + run: cargo fmt --all -- --check feature-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true + - uses: actions/checkout@v3 + - name: Install Protobuf Compiler + run: sudo apt-get install protobuf-compiler + - name: Install Rust Stable + run: rustup default stable - name: Install cargo-hack - uses: actions-rs/cargo@v1 + uses: taiki-e/install-action@v2 with: - command: install - args: cargo-hack + tool: cargo-hack - name: Check Feature Matrix - uses: actions-rs/cargo@v1 - with: - command: hack - args: build --all --all-targets --feature-powerset + run: cargo hack check --all --all-targets --feature-powerset --release test: - name: Test ${{ matrix.rust_version }}/${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Test ${{ matrix.rust_version }} + runs-on: ubuntu-latest strategy: matrix: - rust_version: ['1.55.0', 'stable', 'nightly'] - os: [ubuntu-latest, windows-latest, macOS-latest] + rust_version: ['1.61.0', 'stable', 'nightly'] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 + - name: Install Protobuf Compiler + run: sudo apt-get install protobuf-compiler - name: Install Rust ${{ matrix.rust_version }} - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.rust_version }} - override: true + run: rustup default ${{ matrix.rust_version }} - name: Run Tests - uses: actions-rs/cargo@v1 - with: - command: test - args: --all-features --workspace --exclude=metrics-observer + run: cargo test --all-features --workspace --exclude=metrics-observer -- --test-threads=1 docs: runs-on: ubuntu-latest env: RUSTDOCFLAGS: -Dwarnings steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Install Protobuf Compiler + run: sudo apt-get install protobuf-compiler - name: Install Rust Nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - override: true - components: rust-docs + run: rustup default nightly - name: Check Docs - uses: actions-rs/cargo@v1 - with: - command: doc - args: --all-features --workspace --exclude=metrics-observer --no-deps + run: cargo doc --all-features --workspace --exclude=metrics-observer --no-deps bench: - name: Bench ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + name: Bench + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 + - name: Install Protobuf Compiler + run: sudo apt-get install protobuf-compiler - name: Install Rust Stable - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true + run: rustup default stable - name: Run Benchmarks - uses: actions-rs/cargo@v1 - with: - command: bench - args: --all-features --workspace --exclude=metrics-observer + run: cargo bench --all-features --workspace --exclude=metrics-observer + clippy: + name: Clippy ${{ matrix.rust_version }} + runs-on: ubuntu-latest + strategy: + matrix: + rust_version: ['1.61.0', 'stable', 'nightly'] + steps: + - uses: actions/checkout@v3 + - name: Install Protobuf Compiler + run: sudo apt-get install protobuf-compiler + - name: Install Rust ${{ matrix.rust_version }} + run: rustup default ${{ matrix.rust_version }} + - name: Run Clippy + run: cargo clippy --all-features --workspace --exclude=metrics-observer diff --git a/Cargo.toml b/Cargo.toml index 7911ccd4..66041294 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ [workspace] members = [ "metrics", - "metrics-macros", "metrics-util", "metrics-exporter-tcp", "metrics-exporter-prometheus", diff --git a/README.md b/README.md index a575fc8b..fafdce31 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,6 @@ If you're a library author, you'll only care about using [`metrics`][metrics] to Overall, this repository is home to the following crates: * [`metrics`][metrics]: A lightweight metrics facade, similar to [`log`][log]. -* [`metrics-macros`][metrics-macros]: Procedural macros that power `metrics`. * [`metrics-tracing-context`][metrics-tracing-context]: Allow capturing [`tracing`][tracing] span fields as metric labels. * [`metrics-exporter-tcp`][metrics-exporter-tcp]: A `metrics`-compatible exporter for serving metrics over TCP. @@ -56,17 +55,39 @@ Overall, this repository is home to the following crates: serving a Prometheus scrape endpoint. * [`metrics-util`][metrics-util]: Helper types/functions used by the `metrics` ecosystem. +# community integrations + +As well, there are also some community-maintained exporters and other integrations: + +* [`metrics-exporter-statsd`][metrics-exporter-statsd]: A `metrics`-compatible exporter for sending metrics via StatsD. +* [`metrics-exporter-newrelic`][metrics-exporter-newrelic]: A `metrics`-compatible exporter for sending metrics to New Relic. +* [`opinionated_metrics`][opinionated-metrics]: Opinionated interface to emitting metrics for CLI/server applications, based on `metrics`. + +## MSRV and MSRV policy + +Minimum supported Rust version (MSRV) is currently **1.61.0**, enforced by CI. + +`metrics` will always support _at least_ the latest four versions of stable Rust, based on minor +version releases, and excluding patch versions. Overall, we strive to support older versions where +possible, which means that we generally try to avoid staying up-to-date with every single dependency +(except for security/correctness reasons) and avoid bumping the MSRV just to get access to new +helper methods in the standard library, and so on. + # contributing -We're always looking for users who have thoughts on how to make `metrics` better, or users with interesting use cases. Of course, we're also happy to accept code contributions for outstanding feature requests! 😀 +To those of you who have already contributed to `metrics` in some way, shape, or form: **a big, and continued, "thank you!"** ❤️ + +To everyone else that we haven't had the pleasure of interacting with: we're always looking for thoughts on how to make `metrics` better, or users with interesting use cases. Of course, we're also happy to accept code contributions for outstanding feature requests directly. 😀 -We'd love to chat about any of the above, or anything else, really! You can find us over on [Discord](https://discord.gg/eTwKyY9). +We'd love to chat about any of the above, or anything else related to metrics. Don't hesitate to file an issue on the repository, or come and chat with us over on [Discord](https://discord.gg/eTwKyY9). [metrics]: https://github.com/metrics-rs/metrics/tree/main/metrics -[metrics-macros]: https://github.com/metrics-rs/metrics/tree/main/metrics-macros [metrics-tracing-context]: https://github.com/metrics-rs/metrics/tree/main/metrics-tracing-context [metrics-exporter-tcp]: https://github.com/metrics-rs/metrics/tree/main/metrics-exporter-tcp [metrics-exporter-prometheus]: https://github.com/metrics-rs/metrics/tree/main/metrics-exporter-prometheus [metrics-util]: https://github.com/metrics-rs/metrics/tree/main/metrics-util [log]: https://docs.rs/log [tracing]: https://tracing.rs +[metrics-exporter-statsd]: https://docs.rs/metrics-exporter-statsd +[metrics-exporter-newrelic]: https://docs.rs/metrics-exporter-newrelic +[opinionated-metrics]: https://docs.rs/opinionated_metrics diff --git a/assets/splash.png b/assets/splash.png index 5024edd2..cc47b657 100644 Binary files a/assets/splash.png and b/assets/splash.png differ diff --git a/metrics-benchmark/Cargo.toml b/metrics-benchmark/Cargo.toml index 60da9a08..e48d98fc 100644 --- a/metrics-benchmark/Cargo.toml +++ b/metrics-benchmark/Cargo.toml @@ -3,14 +3,15 @@ name = "metrics-benchmark" version = "0.1.1-alpha.5" authors = ["Toby Lawrence "] edition = "2018" +rust-version = "1.61.0" publish = false [dependencies] log = "0.4" -pretty_env_logger = "0.4" +pretty_env_logger = "0.5" getopts = "0.2" hdrhistogram = { version = "7.2", default-features = false } -quanta = "0.9.3" -atomic-shim = "0.2" -metrics = { version = "^0.18", path = "../metrics" } -metrics-util = { version = "^0.12", path = "../metrics-util" } +quanta = "0.12" +portable-atomic = { version = "1", default-features = false, features = ["fallback"] } +metrics = { version = "0.21", path = "../metrics" } +metrics-util = { version = "0.15", path = "../metrics-util" } diff --git a/metrics-benchmark/src/main.rs b/metrics-benchmark/src/main.rs index a9d5b56a..df2b58cb 100644 --- a/metrics-benchmark/src/main.rs +++ b/metrics-benchmark/src/main.rs @@ -1,12 +1,12 @@ -use atomic_shim::AtomicU64; use getopts::Options; use hdrhistogram::Histogram as HdrHistogram; use log::{error, info}; use metrics::{ - gauge, histogram, increment_counter, register_counter, register_gauge, register_histogram, - Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit, + counter, gauge, histogram, Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, + SharedString, Unit, }; use metrics_util::registry::{AtomicStorage, Registry}; +use portable_atomic::AtomicU64; use quanta::{Clock, Instant as QuantaInstant}; use std::{ env, @@ -62,21 +62,21 @@ impl BenchmarkingRecorder { } impl Recorder for BenchmarkingRecorder { - fn describe_counter(&self, _: KeyName, _: Option, _: &'static str) {} + fn describe_counter(&self, _: KeyName, _: Option, _: SharedString) {} - fn describe_gauge(&self, _: KeyName, _: Option, _: &'static str) {} + fn describe_gauge(&self, _: KeyName, _: Option, _: SharedString) {} - fn describe_histogram(&self, _: KeyName, _: Option, _: &'static str) {} + fn describe_histogram(&self, _: KeyName, _: Option, _: SharedString) {} - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { self.registry.get_or_create_counter(key, |c| Counter::from_arc(c.clone())) } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { self.registry.get_or_create_gauge(key, |g| Gauge::from_arc(g.clone())) } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { self.registry.get_or_create_histogram(key, |h| Histogram::from_arc(h.clone())) } } @@ -120,9 +120,9 @@ impl Generator { if let Some(t0) = self.t0 { let start = if loop_counter % LOOP_SAMPLE == 0 { Some(clock.now()) } else { None }; - increment_counter!("ok"); - gauge!("total", self.gauge as f64); - histogram!("ok", t1.sub(t0)); + counter!("ok").increment(1); + gauge!("total").set(self.gauge as f64); + histogram!("ok").record(t1.sub(t0)); if let Some(val) = start { let delta = clock.now() - val; @@ -145,9 +145,9 @@ impl Generator { let clock = Clock::new(); let mut loop_counter = 0; - let counter = register_counter!("ok"); - let gauge = register_gauge!("total"); - let histogram = register_histogram!("ok"); + let counter = counter!("ok"); + let gauge = gauge!("total"); + let histogram = histogram!("ok"); loop { loop_counter += 1; diff --git a/metrics-exporter-prometheus/CHANGELOG.md b/metrics-exporter-prometheus/CHANGELOG.md index 0638bace..600265f2 100644 --- a/metrics-exporter-prometheus/CHANGELOG.md +++ b/metrics-exporter-prometheus/CHANGELOG.md @@ -8,19 +8,70 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - ReleaseDate +### Added + +- Support for using HTTPS in Push Gateway mode. ([#392](https://github.com/metrics-rs/metrics/pull/392)) + +## [0.12.2] - 2023-12-13 + +### Fixed + +- Fixed overflow/underflow panic with time moving backwards ([#423](https://github.com/metrics-rs/metrics/pull/423)) + +## [0.12.1] - 2023-05-09 + +### Added + +- Support for specifying a username/password for HTTP Basic Authentication when pushing to a Push + Gateway. ([#366](https://github.com/metrics-rs/metrics/pull/366)) + +## [0.12.0] - 2023-04-16 + +### Changed + +- Bump MSRV to 1.61.0. +- Switch to `metrics`-exposed version of `AtomicU64`. + +## [0.11.0] - 2022-07-20 + +### Changed + +- Aggregated summaries are now rolling, allowing oldering data points to expire and quantile values + to reflect the recent past rather than the lifetime of a histogram. + ([#306](https://github.com/metrics-rs/metrics/pull/306)) + + They have a default width of three buckets, with each bucket being 20 seconds wide. This means + only the last 60 seconds of a histogram -- in 20 second granularity -- will contribute to the + quantiles emitted. + + We'll expose the ability to tune these values in the future. +- Switched to using `portable_atomic` for 64-bit atomics on more architectures. + ([#313](https://github.com/metrics-rs/metrics/pull/313)) + + +## [0.10.0] - 2022-05-30 + +### Fixed + +- In some cases, metric names were being "sanitized" when they were already valid. + ([#290](https://github.com/metrics-rs/metrics/pull/290), [#296](https://github.com/metrics-rs/metrics/pull/296)) + ## [0.9.0] - 2022-03-10 -## Added +### Added + - New top-level module, `formatting`, which exposes many of the helper methods used to sanitize and render the actual Prometheus exposition format. ([#285](https://github.com/metrics-rs/metrics/pull/285)) ## [0.8.0] - 2022-01-14 ### Added + - New builder method, `PrometheusBuilder::install_recorder`, which builds and installs the recorder and returns a `PrometheusHandle` that can be used to interact with the recorder. ### Changed + - Updated various dependencies in order to properly scope dependencies to only the necessary feature flags, and thus optimize build times and reduce transitive dependencies. - Updated to the new handle-based design of `metrics`. @@ -35,6 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (name/labels) was recorded. ### Fixed + - Label keys and values, as well as metric descriptions, are now correctly sanitized according to the Prometheus [data model](https://prometheus.io/docs/concepts/data_model/) and [exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md). @@ -46,21 +98,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.7.0] - 2021-12-16 ### Changed + - Calling `PrometheusBuilder::install` inside a Tokio runtime will spawn the exporter on that runtime rather than spawning a new runtime. ([#251](https://github.com/metrics-rs/metrics/pull/251)) ## [0.6.1] - 2021-09-16 ### Changed + - Simple release to bump dependencies. ## [0.6.0] - 2021-07-15 ### Added + - Support for pushing to a Push Gateway. ([#217](https://github.com/metrics-rs/metrics/pull/217)) ## [0.5.0] - 2021-05-18 + ### Added + - `PrometheusBuilder::add_allowed`, which enables the exporter to be configured with a list of IP addresses or subnets that are allowed to connect. By default, no restrictions are enforced. @@ -68,16 +125,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.4.0] - 2021-05-03 ### Changed + - Bumped `metrics` dependency to `0.15` and updated the necessary APIs. ## [0.3.0] - 2021-02-02 + ### Changed + - Bumped `metrics` dependency to `0.14`. ## [0.2.0] - 2021-01-23 + ### Changed + - Switched from `MetricKind` for `MetricKindMask` for `PrometheusBuilder::idle_timeout`. ## [0.1.0] - 2021-01-22 + ### Added + - Genesis of the crate. diff --git a/metrics-exporter-prometheus/Cargo.toml b/metrics-exporter-prometheus/Cargo.toml index b9d4d20d..52e0626e 100644 --- a/metrics-exporter-prometheus/Cargo.toml +++ b/metrics-exporter-prometheus/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "metrics-exporter-prometheus" -version = "0.9.0" +version = "0.12.2" authors = ["Toby Lawrence "] edition = "2018" +rust-version = "1.61.0" license = "MIT" @@ -19,23 +20,25 @@ keywords = ["metrics", "telemetry", "prometheus"] default = ["http-listener", "push-gateway"] async-runtime = ["tokio", "hyper"] http-listener = ["async-runtime", "hyper/server", "ipnet"] -push-gateway = ["async-runtime", "hyper/client", "tracing"] +push-gateway = ["async-runtime", "hyper/client", "hyper-tls", "tracing"] [dependencies] +base64 = { version = "0.21.0", default-features = false, features = ["std"] } +indexmap = { version = "2.1", default-features = false } +metrics = { version = "^0.21", path = "../metrics" } +metrics-util = { version = "^0.15", path = "../metrics-util", default-features = false, features = ["recency", "registry", "summary"] } +parking_lot = { version = "0.11", default-features = false } +quanta = { version = "0.12", default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -metrics = { version = "^0.18", path = "../metrics" } -metrics-util = { version = "^0.12", path = "../metrics-util", default-features = false, features = ["recency", "registry", "summary"] } -parking_lot = { version = "0.11", default-features = false } thiserror = { version = "1", default-features = false } -quanta = { version = "0.9.3", default-features = false } -indexmap = { version = "1", default-features = false } # Optional hyper = { version = "0.14", default-features = false, features = ["tcp", "http1"], optional = true } ipnet = { version = "2", optional = true } tokio = { version = "1", features = ["rt", "net", "time"], optional = true } tracing = { version = "0.1.26", optional = true } +hyper-tls = { version = "0.5.0", optional = true } [dev-dependencies] tracing = "0.1" diff --git a/metrics-exporter-prometheus/examples/prometheus_push_gateway.rs b/metrics-exporter-prometheus/examples/prometheus_push_gateway.rs index 6a3ceca5..229a0f38 100644 --- a/metrics-exporter-prometheus/examples/prometheus_push_gateway.rs +++ b/metrics-exporter-prometheus/examples/prometheus_push_gateway.rs @@ -4,10 +4,7 @@ use std::thread; use std::time::Duration; #[allow(unused_imports)] -use metrics::{ - decrement_gauge, gauge, histogram, increment_counter, increment_gauge, register_counter, - register_histogram, -}; +use metrics::{counter, gauge, histogram}; use metrics::{describe_counter, describe_histogram}; #[allow(unused_imports)] use metrics_exporter_prometheus::PrometheusBuilder; @@ -21,7 +18,12 @@ fn main() { tracing_subscriber::fmt::init(); PrometheusBuilder::new() - .with_push_gateway("http://127.0.0.1:9091/metrics/job/example", Duration::from_secs(10)) + .with_push_gateway( + "http://127.0.0.1:9091/metrics/job/example", + Duration::from_secs(10), + None, + None, + ) .expect("push gateway endpoint should be valid") .idle_timeout( MetricKindMask::COUNTER | MetricKindMask::HISTOGRAM, @@ -45,23 +47,24 @@ fn main() { let clock = Clock::new(); let mut last = None; - increment_counter!("idle_metric"); - gauge!("testing", 42.0); + counter!("idle_metric").increment(1); + gauge!("testing").set(42.0); // Loop over and over, pretending to do some work. loop { - increment_counter!("tcp_server_loops", "system" => "foo"); + counter!("tcp_server_loops", "system" => "foo").increment(1); if let Some(t) = last { let delta: Duration = clock.now() - t; - histogram!("tcp_server_loop_delta_secs", delta, "system" => "foo"); + histogram!("tcp_server_loop_delta_secs", "system" => "foo").record(delta); } let increment_gauge = thread_rng().gen_bool(0.75); + let gauge = gauge!("lucky_iterations"); if increment_gauge { - increment_gauge!("lucky_iterations", 1.0); + gauge.increment(1.0); } else { - decrement_gauge!("lucky_iterations", 1.0); + gauge.decrement(1.0); } last = Some(clock.now()); diff --git a/metrics-exporter-prometheus/examples/prometheus_server.rs b/metrics-exporter-prometheus/examples/prometheus_server.rs index 9ff05fd9..13a68a56 100644 --- a/metrics-exporter-prometheus/examples/prometheus_server.rs +++ b/metrics-exporter-prometheus/examples/prometheus_server.rs @@ -1,10 +1,7 @@ use std::thread; use std::time::Duration; -use metrics::{ - decrement_gauge, describe_counter, describe_histogram, gauge, histogram, increment_counter, - increment_gauge, -}; +use metrics::{counter, describe_counter, describe_histogram, gauge, histogram}; use metrics_exporter_prometheus::PrometheusBuilder; use metrics_util::MetricKindMask; @@ -38,23 +35,24 @@ fn main() { let clock = Clock::new(); let mut last = None; - increment_counter!("idle_metric"); - gauge!("testing", 42.0); + counter!("idle_metric").increment(1); + gauge!("testing").set(42.0); // Loop over and over, pretending to do some work. loop { - increment_counter!("tcp_server_loops", "system" => "foo"); + counter!("tcp_server_loops", "system" => "foo").increment(1); if let Some(t) = last { let delta: Duration = clock.now() - t; - histogram!("tcp_server_loop_delta_secs", delta, "system" => "foo"); + histogram!("tcp_server_loop_delta_secs", "system" => "foo").record(delta); } let increment_gauge = thread_rng().gen_bool(0.75); + let gauge = gauge!("lucky_iterations"); if increment_gauge { - increment_gauge!("lucky_iterations", 1.0); + gauge.increment(1.0); } else { - decrement_gauge!("lucky_iterations", 1.0); + gauge.decrement(1.0); } last = Some(clock.now()); diff --git a/metrics-exporter-prometheus/src/builder.rs b/metrics-exporter-prometheus/src/builder.rs index 33d37040..70f5f762 100644 --- a/metrics-exporter-prometheus/src/builder.rs +++ b/metrics-exporter-prometheus/src/builder.rs @@ -7,6 +7,7 @@ use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; #[cfg(any(feature = "http-listener", feature = "push-gateway"))] use std::pin::Pin; +use std::sync::RwLock; #[cfg(any(feature = "http-listener", feature = "push-gateway"))] use std::thread; use std::time::Duration; @@ -25,13 +26,15 @@ use hyper::{ use hyper::{ body::{aggregate, Buf}, client::Client, + http::HeaderValue, Method, Request, Uri, }; +#[cfg(feature = "push-gateway")] +use hyper_tls::HttpsConnector; use indexmap::IndexMap; #[cfg(feature = "http-listener")] use ipnet::IpNet; -use parking_lot::RwLock; use quanta::Clock; #[cfg(any(feature = "http-listener", feature = "push-gateway"))] use tokio::runtime; @@ -47,6 +50,7 @@ use metrics_util::{ use crate::common::Matcher; use crate::distribution::DistributionBuilder; use crate::recorder::{Inner, PrometheusRecorder}; +use crate::registry::AtomicStorage; use crate::{common::BuildError, PrometheusHandle}; #[cfg(any(feature = "http-listener", feature = "push-gateway"))] @@ -61,7 +65,12 @@ enum ExporterConfig { // Run a push gateway task sending to the given `endpoint` after `interval` time has elapsed, // infinitely. #[cfg(feature = "push-gateway")] - PushGateway { endpoint: Uri, interval: Duration }, + PushGateway { + endpoint: Uri, + interval: Duration, + username: Option, + password: Option, + }, #[allow(dead_code)] Unconfigured, @@ -156,6 +165,8 @@ impl PrometheusBuilder { mut self, endpoint: T, interval: Duration, + username: Option, + password: Option, ) -> Result where T: AsRef, @@ -164,6 +175,8 @@ impl PrometheusBuilder { endpoint: Uri::try_from(endpoint.as_ref()) .map_err(|e| BuildError::InvalidPushGatewayEndpoint(e.to_string()))?, interval, + username, + password, }; Ok(self) @@ -391,6 +404,7 @@ impl PrometheusBuilder { /// /// If there is an error while building the recorder and exporter, an error variant will be /// returned describing the error. + #[warn(clippy::too_many_lines)] #[cfg(any(feature = "http-listener", feature = "push-gateway"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "http-listener", feature = "push-gateway"))))] #[cfg_attr(not(feature = "http-listener"), allow(unused_mut))] @@ -447,16 +461,23 @@ impl PrometheusBuilder { } #[cfg(feature = "push-gateway")] - ExporterConfig::PushGateway { endpoint, interval } => { + ExporterConfig::PushGateway { endpoint, interval, username, password } => { let exporter = async move { - let client = Client::new(); + let https = HttpsConnector::new(); + let client = Client::builder().build::<_, hyper::Body>(https); + let auth = username.as_ref().map(|name| basic_auth(name, password.as_deref())); loop { // Sleep for `interval` amount of time, and then do a push. tokio::time::sleep(interval).await; + let mut builder = Request::builder(); + if let Some(auth) = &auth { + builder = builder.header("authorization", auth.clone()); + } + let output = handle.render(); - let result = Request::builder() + let result = builder .method(Method::PUT) .uri(endpoint.clone()) .body(Body::from(output)); @@ -479,9 +500,9 @@ impl PrometheusBuilder { let body = body .map_err(|_| ()) .map(|mut b| b.copy_to_bytes(b.remaining())) - .map(|b| (&b[..]).to_vec()) + .map(|b| b[..].to_vec()) .and_then(|s| String::from_utf8(s).map_err(|_| ())) - .unwrap_or_else(|_| { + .unwrap_or_else(|()| { String::from("") }); error!( @@ -508,7 +529,7 @@ impl PrometheusBuilder { pub(crate) fn build_with_clock(self, clock: Clock) -> PrometheusRecorder { let inner = Inner { - registry: Registry::new(GenerationalStorage::atomic()), + registry: Registry::new(GenerationalStorage::new(AtomicStorage)), recency: Recency::new(clock, self.recency_mask, self.idle_timeout), distributions: RwLock::new(HashMap::new()), distribution_builder: DistributionBuilder::new( @@ -530,6 +551,25 @@ impl Default for PrometheusBuilder { } } +#[cfg(feature = "push-gateway")] +fn basic_auth(username: &str, password: Option<&str>) -> HeaderValue { + use base64::prelude::BASE64_STANDARD; + use base64::write::EncoderWriter; + use std::io::Write; + + let mut buf = b"Basic ".to_vec(); + { + let mut encoder = EncoderWriter::new(&mut buf, &BASE64_STANDARD); + let _ = write!(encoder, "{username}:"); + if let Some(password) = password { + let _ = write!(encoder, "{password}"); + } + } + let mut header = HeaderValue::from_bytes(&buf).expect("base64 is always valid HeaderValue"); + header.set_sensitive(true); + header +} + #[cfg(test)] mod tests { use std::time::Duration; @@ -541,12 +581,16 @@ mod tests { use super::{Matcher, PrometheusBuilder}; + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); + #[test] fn test_render() { - let recorder = PrometheusBuilder::new().build_recorder(); + let recorder = + PrometheusBuilder::new().set_quantiles(&[0.0, 1.0]).unwrap().build_recorder(); let key = Key::from_name("basic_counter"); - let counter1 = recorder.register_counter(&key); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(42); let handle = recorder.handle(); @@ -557,7 +601,7 @@ mod tests { let labels = vec![Label::new("wutang", "forever")]; let key = Key::from_parts("basic_gauge", labels); - let gauge1 = recorder.register_gauge(&key); + let gauge1 = recorder.register_gauge(&key, &METADATA); gauge1.set(-3.14); let rendered = handle.render(); let expected_gauge = format!( @@ -568,18 +612,13 @@ mod tests { assert_eq!(rendered, expected_gauge); let key = Key::from_name("basic_histogram"); - let histogram1 = recorder.register_histogram(&key); + let histogram1 = recorder.register_histogram(&key, &METADATA); histogram1.record(12.0); let rendered = handle.render(); let histogram_data = concat!( "# TYPE basic_histogram summary\n", "basic_histogram{quantile=\"0\"} 12\n", - "basic_histogram{quantile=\"0.5\"} 12\n", - "basic_histogram{quantile=\"0.9\"} 12\n", - "basic_histogram{quantile=\"0.95\"} 12\n", - "basic_histogram{quantile=\"0.99\"} 12\n", - "basic_histogram{quantile=\"0.999\"} 12\n", "basic_histogram{quantile=\"1\"} 12\n", "basic_histogram_sum 12\n", "basic_histogram_count 1\n", @@ -615,19 +654,19 @@ mod tests { .build_recorder(); let full_key = Key::from_name("metrics.testing_foo"); - let full_key_histo = recorder.register_histogram(&full_key); + let full_key_histo = recorder.register_histogram(&full_key, &METADATA); full_key_histo.record(FULL_VALUES[0]); let prefix_key = Key::from_name("metrics.testing_bar"); - let prefix_key_histo = recorder.register_histogram(&prefix_key); + let prefix_key_histo = recorder.register_histogram(&prefix_key, &METADATA); prefix_key_histo.record(PREFIX_VALUES[1]); let suffix_key = Key::from_name("metrics_testin_foo"); - let suffix_key_histo = recorder.register_histogram(&suffix_key); + let suffix_key_histo = recorder.register_histogram(&suffix_key, &METADATA); suffix_key_histo.record(SUFFIX_VALUES[2]); let default_key = Key::from_name("metrics.wee"); - let default_key_histo = recorder.register_histogram(&default_key); + let default_key_histo = recorder.register_histogram(&default_key, &METADATA); default_key_histo.record(DEFAULT_VALUES[2] + 1.0); let full_data = concat!( @@ -685,18 +724,20 @@ mod tests { let recorder = PrometheusBuilder::new() .idle_timeout(MetricKindMask::ALL, Some(Duration::from_secs(10))) + .set_quantiles(&[0.0, 1.0]) + .unwrap() .build_with_clock(clock); let key = Key::from_name("basic_counter"); - let counter1 = recorder.register_counter(&key); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(42); let key = Key::from_name("basic_gauge"); - let gauge1 = recorder.register_gauge(&key); + let gauge1 = recorder.register_gauge(&key, &METADATA); gauge1.set(-3.14); let key = Key::from_name("basic_histogram"); - let histo1 = recorder.register_histogram(&key); + let histo1 = recorder.register_histogram(&key, &METADATA); histo1.record(1.0); let handle = recorder.handle(); @@ -708,11 +749,6 @@ mod tests { "basic_gauge -3.14\n\n", "# TYPE basic_histogram summary\n", "basic_histogram{quantile=\"0\"} 1\n", - "basic_histogram{quantile=\"0.5\"} 1\n", - "basic_histogram{quantile=\"0.9\"} 1\n", - "basic_histogram{quantile=\"0.95\"} 1\n", - "basic_histogram{quantile=\"0.99\"} 1\n", - "basic_histogram{quantile=\"0.999\"} 1\n", "basic_histogram{quantile=\"1\"} 1\n", "basic_histogram_sum 1\n", "basic_histogram_count 1\n\n", @@ -738,18 +774,20 @@ mod tests { MetricKindMask::COUNTER | MetricKindMask::HISTOGRAM, Some(Duration::from_secs(10)), ) + .set_quantiles(&[0.0, 1.0]) + .unwrap() .build_with_clock(clock); let key = Key::from_name("basic_counter"); - let counter1 = recorder.register_counter(&key); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(42); let key = Key::from_name("basic_gauge"); - let gauge1 = recorder.register_gauge(&key); + let gauge1 = recorder.register_gauge(&key, &METADATA); gauge1.set(-3.14); let key = Key::from_name("basic_histogram"); - let histo1 = recorder.register_histogram(&key); + let histo1 = recorder.register_histogram(&key, &METADATA); histo1.record(1.0); let handle = recorder.handle(); @@ -761,11 +799,6 @@ mod tests { "basic_gauge -3.14\n\n", "# TYPE basic_histogram summary\n", "basic_histogram{quantile=\"0\"} 1\n", - "basic_histogram{quantile=\"0.5\"} 1\n", - "basic_histogram{quantile=\"0.9\"} 1\n", - "basic_histogram{quantile=\"0.95\"} 1\n", - "basic_histogram{quantile=\"0.99\"} 1\n", - "basic_histogram{quantile=\"0.999\"} 1\n", "basic_histogram{quantile=\"1\"} 1\n", "basic_histogram_sum 1\n", "basic_histogram_count 1\n\n", @@ -790,18 +823,20 @@ mod tests { let recorder = PrometheusBuilder::new() .idle_timeout(MetricKindMask::ALL, Some(Duration::from_secs(10))) + .set_quantiles(&[0.0, 1.0]) + .unwrap() .build_with_clock(clock); let key = Key::from_name("basic_counter"); - let counter1 = recorder.register_counter(&key); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(42); let key = Key::from_name("basic_gauge"); - let gauge1 = recorder.register_gauge(&key); + let gauge1 = recorder.register_gauge(&key, &METADATA); gauge1.set(-3.14); let key = Key::from_name("basic_histogram"); - let histo1 = recorder.register_histogram(&key); + let histo1 = recorder.register_histogram(&key, &METADATA); histo1.record(1.0); let handle = recorder.handle(); @@ -813,11 +848,6 @@ mod tests { "basic_gauge -3.14\n\n", "# TYPE basic_histogram summary\n", "basic_histogram{quantile=\"0\"} 1\n", - "basic_histogram{quantile=\"0.5\"} 1\n", - "basic_histogram{quantile=\"0.9\"} 1\n", - "basic_histogram{quantile=\"0.95\"} 1\n", - "basic_histogram{quantile=\"0.99\"} 1\n", - "basic_histogram{quantile=\"0.999\"} 1\n", "basic_histogram{quantile=\"1\"} 1\n", "basic_histogram_sum 1\n", "basic_histogram_count 1\n\n", @@ -830,7 +860,7 @@ mod tests { assert_eq!(rendered, expected); let key = Key::from_parts("basic_histogram", vec![Label::new("type", "special")]); - let histo2 = recorder.register_histogram(&key); + let histo2 = recorder.register_histogram(&key, &METADATA); histo2.record(2.0); let expected_second = concat!( @@ -840,20 +870,10 @@ mod tests { "basic_gauge -3.14\n\n", "# TYPE basic_histogram summary\n", "basic_histogram{quantile=\"0\"} 1\n", - "basic_histogram{quantile=\"0.5\"} 1\n", - "basic_histogram{quantile=\"0.9\"} 1\n", - "basic_histogram{quantile=\"0.95\"} 1\n", - "basic_histogram{quantile=\"0.99\"} 1\n", - "basic_histogram{quantile=\"0.999\"} 1\n", "basic_histogram{quantile=\"1\"} 1\n", "basic_histogram_sum 1\n", "basic_histogram_count 1\n", "basic_histogram{type=\"special\",quantile=\"0\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.5\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.9\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.95\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.99\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.999\"} 2\n", "basic_histogram{type=\"special\",quantile=\"1\"} 2\n", "basic_histogram_sum{type=\"special\"} 2\n", "basic_histogram_count{type=\"special\"} 1\n\n", @@ -864,11 +884,6 @@ mod tests { let expected_after = concat!( "# TYPE basic_histogram summary\n", "basic_histogram{type=\"special\",quantile=\"0\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.5\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.9\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.95\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.99\"} 2\n", - "basic_histogram{type=\"special\",quantile=\"0.999\"} 2\n", "basic_histogram{type=\"special\",quantile=\"1\"} 2\n", "basic_histogram_sum{type=\"special\"} 2\n", "basic_histogram_count{type=\"special\"} 1\n\n", @@ -888,11 +903,11 @@ mod tests { .build_with_clock(clock); let key = Key::from_name("basic_counter"); - let counter1 = recorder.register_counter(&key); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(42); let key = Key::from_name("basic_gauge"); - let gauge1 = recorder.register_gauge(&key); + let gauge1 = recorder.register_gauge(&key, &METADATA); gauge1.set(-3.14); let handle = recorder.handle(); @@ -937,7 +952,7 @@ mod tests { .build_with_clock(clock); let key = Key::from_name("basic_counter"); - let counter1 = recorder.register_counter(&key); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(42); // First render, which starts tracking the counter in the recency state. @@ -976,7 +991,7 @@ mod tests { .add_global_label("foo", "bar") .build_recorder(); let key = Key::from_name("basic_counter"); - let counter1 = recorder.register_counter(&key); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(42); let handle = recorder.handle(); @@ -992,7 +1007,7 @@ mod tests { let key = Key::from_name("overridden").with_extra_labels(vec![Label::new("foo", "overridden")]); - let counter1 = recorder.register_counter(&key); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(1); let handle = recorder.handle(); @@ -1009,8 +1024,8 @@ mod tests { let key_name = KeyName::from("yee_haw:lets go"); let key = Key::from_name(key_name.clone()) .with_extra_labels(vec![Label::new("øhno", "\"yeet\nies\\\"")]); - recorder.describe_counter(key_name, None, "\"Simplë stuff.\nRëally.\""); - let counter1 = recorder.register_counter(&key); + recorder.describe_counter(key_name, None, "\"Simplë stuff.\nRëally.\"".into()); + let counter1 = recorder.register_counter(&key, &METADATA); counter1.increment(1); let handle = recorder.handle(); @@ -1020,3 +1035,39 @@ mod tests { assert_eq!(rendered, expected_counter); } } + +#[cfg(all(test, feature = "push-gateway"))] +mod push_gateway_tests { + use crate::builder::basic_auth; + + #[test] + pub fn test_basic_auth() { + use base64::prelude::BASE64_STANDARD; + use base64::read::DecoderReader; + use std::io::Read; + + const BASIC: &str = "Basic "; + + // username only + let username = "metrics"; + let header = basic_auth(username, None); + + let reader = &header.as_ref()[BASIC.len()..]; + let mut decoder = DecoderReader::new(reader, &BASE64_STANDARD); + let mut result = Vec::new(); + decoder.read_to_end(&mut result).unwrap(); + assert_eq!(b"metrics:", &result[..]); + assert!(header.is_sensitive()); + + // username/password + let password = "123!_@ABC"; + let header = basic_auth(username, Some(password)); + + let reader = &header.as_ref()[BASIC.len()..]; + let mut decoder = DecoderReader::new(reader, &BASE64_STANDARD); + let mut result = Vec::new(); + decoder.read_to_end(&mut result).unwrap(); + assert_eq!(b"metrics:123!_@ABC", &result[..]); + assert!(header.is_sensitive()); + } +} diff --git a/metrics-exporter-prometheus/src/distribution.rs b/metrics-exporter-prometheus/src/distribution.rs index 06f7478b..24475d82 100644 --- a/metrics-exporter-prometheus/src/distribution.rs +++ b/metrics-exporter-prometheus/src/distribution.rs @@ -1,5 +1,9 @@ +use std::num::NonZeroU32; +use std::time::Duration; use std::{collections::HashMap, sync::Arc}; +use quanta::Instant; + use crate::common::Matcher; use metrics_util::{Histogram, Quantile, Summary}; @@ -18,11 +22,12 @@ pub enum Distribution { /// Computes and exposes value quantiles directly to Prometheus i.e. 50% of /// requests were faster than 200ms, and 99% of requests were faster than /// 1000ms, etc. - Summary(Summary, Arc>, f64), + Summary(RollingSummary, Arc>, f64), } impl Distribution { /// Creates a histogram distribution. + #[warn(clippy::missing_panics_doc)] pub fn new_histogram(buckets: &[f64]) -> Distribution { let hist = Histogram::new(buckets).expect("buckets should never be empty"); Distribution::Histogram(hist) @@ -30,17 +35,19 @@ impl Distribution { /// Creates a summary distribution. pub fn new_summary(quantiles: Arc>) -> Distribution { - let summary = Summary::with_defaults(); + let summary = RollingSummary::default(); Distribution::Summary(summary, quantiles, 0.0) } /// Records the given `samples` in the current distribution. - pub fn record_samples(&mut self, samples: &[f64]) { + pub fn record_samples(&mut self, samples: &[(f64, Instant)]) { match self { - Distribution::Histogram(hist) => hist.record_many(samples), + Distribution::Histogram(hist) => { + hist.record_many(samples.iter().map(|(sample, _ts)| sample)); + } Distribution::Summary(hist, _, sum) => { - for sample in samples { - hist.add(*sample); + for (sample, ts) in samples { + hist.add(*sample, *ts); *sum += *sample; } } @@ -77,7 +84,7 @@ impl DistributionBuilder { /// Returns a distribution for the given metric key. pub fn get_distribution(&self, name: &str) -> Distribution { if let Some(ref overrides) = self.bucket_overrides { - for (matcher, buckets) in overrides.iter() { + for (matcher, buckets) in overrides { if matcher.matches(name) { return Distribution::new_histogram(buckets); } @@ -98,7 +105,7 @@ impl DistributionBuilder { } if let Some(ref overrides) = self.bucket_overrides { - for (matcher, _) in overrides.iter() { + for (matcher, _) in overrides { if matcher.matches(name) { return "histogram"; } @@ -108,3 +115,268 @@ impl DistributionBuilder { "summary" } } + +#[derive(Clone)] +struct Bucket { + begin: Instant, + summary: Summary, +} + +/// A `RollingSummary` manages a list of [Summary] so that old results can be expired. +#[derive(Clone)] +pub struct RollingSummary { + // Buckets are ordered with the latest buckets first. The buckets are kept in alignment based + // on the instant of the first added bucket and the bucket_duration. There may be gaps in the + // bucket list. + buckets: Vec, + // Maximum number of buckets to track. + max_buckets: usize, + // Duration of values stored per bucket. + bucket_duration: Duration, + // This is the maximum duration a bucket will be kept. + max_bucket_duration: Duration, + // Total samples since creation of this summary. This is separate from the Summary since it is + // never reset. + count: usize, +} + +impl Default for RollingSummary { + fn default() -> Self { + RollingSummary::new(NonZeroU32::new(3).unwrap(), Duration::from_secs(20)) + } +} + +impl RollingSummary { + /// Create a new `RollingSummary` with the given number of `buckets` and `bucket-duration`. + /// + /// The summary will store quantiles over `buckets * bucket_duration` seconds. + pub fn new(buckets: std::num::NonZeroU32, bucket_duration: Duration) -> RollingSummary { + assert!(!bucket_duration.is_zero()); + let max_bucket_duration = bucket_duration * buckets.get(); + let max_buckets = buckets.get() as usize; + + RollingSummary { + buckets: Vec::with_capacity(max_buckets), + max_buckets, + bucket_duration, + max_bucket_duration, + count: 0, + } + } + + /// Add a sample `value` to the `RollingSummary` at the time `now`. + /// + /// Any values that expire at the `value_ts` are removed from the `RollingSummary`. + pub fn add(&mut self, value: f64, now: Instant) { + // The count is incremented even if this value is too old to be saved in any bucket. + self.count += 1; + + // If we can find a bucket that this value belongs in, then we can just add it in and be + // done. + for bucket in &mut self.buckets { + let end = bucket.begin + self.bucket_duration; + + // If this value belongs in a future bucket... + if now > bucket.begin + self.bucket_duration { + break; + } + + if now >= bucket.begin && now < end { + bucket.summary.add(value); + return; + } + } + + // Remove any expired buckets. + if let Some(cutoff) = now.checked_sub(self.max_bucket_duration) { + self.buckets.retain(|b| b.begin > cutoff); + } + + if self.buckets.is_empty() { + let mut summary = Summary::with_defaults(); + summary.add(value); + self.buckets.push(Bucket { begin: now, summary }); + return; + } + + // Take the first bucket time as a reference. Other buckets will be created at an offset + // of this time. We know this time is close to the value_ts, if it were much older the + // bucket would have been removed. + let reftime = self.buckets[0].begin; + + let mut summary = Summary::with_defaults(); + summary.add(value); + + // If the value is newer than the first bucket then count upwards to the new bucket time. + let mut begin; + if now > reftime { + begin = reftime + self.bucket_duration; + let mut end = begin + self.bucket_duration; + while now < begin || now >= end { + begin += self.bucket_duration; + end += self.bucket_duration; + } + + self.buckets.truncate(self.max_buckets - 1); + self.buckets.insert(0, Bucket { begin, summary }); + } + } + + /// Return a merged Summary of all items that are valid at `now`. + /// + /// # Warning + /// + /// The snapshot `Summary::count()` contains the total number of values considered in the + /// Snapshot, which is not the full count of the `RollingSummary`. Use `RollingSummary::count()` + /// instead. + pub fn snapshot(&self, now: Instant) -> Summary { + let cutoff = now.checked_sub(self.max_bucket_duration); + let mut acc = Summary::with_defaults(); + self.buckets + .iter() + .filter(|b| if let Some(cutoff) = cutoff { b.begin > cutoff } else { true }) + .map(|b| &b.summary) + .fold(&mut acc, |acc, item| { + acc.merge(item).expect("merge can only fail if summary config inconsistent"); + acc + }); + acc + } + + /// Whether or not this summary is empty. + pub fn is_empty(&self) -> bool { + self.count() == 0 + } + + /// Gets the totoal number of samples this summary has seen so far. + pub fn count(&self) -> usize { + self.count + } + + #[cfg(test)] + fn buckets(&self) -> &Vec { + &self.buckets + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use quanta::Clock; + + #[test] + fn new_rolling_summary() { + let summary = RollingSummary::default(); + + assert_eq!(0, summary.buckets().len()); + assert_eq!(0, summary.count()); + assert!(summary.is_empty()); + } + + #[test] + fn empty_snapshot() { + let (clock, _mock) = Clock::mock(); + let summary = RollingSummary::default(); + let snapshot = summary.snapshot(clock.now()); + + assert_eq!(0, snapshot.count()); + assert_eq!(f64::INFINITY, snapshot.min()); + assert_eq!(f64::NEG_INFINITY, snapshot.max()); + assert_eq!(None, snapshot.quantile(0.5)); + } + + #[test] + fn snapshot() { + let (clock, mock) = Clock::mock(); + mock.increment(Duration::from_secs(3600)); + + let mut summary = RollingSummary::default(); + summary.add(42.0, clock.now()); + mock.increment(Duration::from_secs(20)); + summary.add(42.0, clock.now()); + mock.increment(Duration::from_secs(20)); + summary.add(42.0, clock.now()); + + let snapshot = summary.snapshot(clock.now()); + + assert_eq!(42.0, snapshot.min()); + assert_eq!(42.0, snapshot.max()); + // 42 +/- (42 * 0.0001) + assert!(Some(41.9958) < snapshot.quantile(0.5)); + assert!(Some(42.0042) > snapshot.quantile(0.5)); + } + + #[test] + fn add_first_value() { + let (clock, mock) = Clock::mock(); + mock.increment(Duration::from_secs(3600)); + + let mut summary = RollingSummary::default(); + summary.add(42.0, clock.now()); + + assert_eq!(1, summary.buckets().len()); + assert_eq!(1, summary.count()); + assert!(!summary.is_empty()); + } + + #[test] + fn add_new_head() { + let (clock, mock) = Clock::mock(); + mock.increment(Duration::from_secs(3600)); + + let mut summary = RollingSummary::default(); + summary.add(42.0, clock.now()); + mock.increment(Duration::from_secs(20)); + summary.add(42.0, clock.now()); + + assert_eq!(2, summary.buckets().len()); + } + + #[test] + fn truncate_old_buckets() { + let (clock, mock) = Clock::mock(); + mock.increment(Duration::from_secs(3600)); + + let mut summary = RollingSummary::default(); + summary.add(42.0, clock.now()); + + for _ in 0..3 { + mock.increment(Duration::from_secs(20)); + summary.add(42.0, clock.now()); + } + + assert_eq!(3, summary.buckets().len()); + } + + #[test] + fn add_value_ts_before_first_bucket() { + let (clock, mock) = Clock::mock(); + mock.increment(Duration::from_secs(4)); + + let bucket_count = NonZeroU32::new(2).unwrap(); + let bucket_width = Duration::from_secs(5); + + let mut summary = RollingSummary::new(bucket_count, bucket_width); + assert_eq!(0, summary.buckets().len()); + assert_eq!(0, summary.count()); + + // Add a single value to create our first bucket. + summary.add(42.0, clock.now()); + + // Make sure the value got added. + assert_eq!(1, summary.buckets().len()); + assert_eq!(1, summary.count()); + assert!(!summary.is_empty()); + + // Our first bucket is now marked as begin=4/width=5, so make sure that if we add a version + // with now=3, the count goes up but it's not actually added. + mock.decrement(Duration::from_secs(1)); + + summary.add(43.0, clock.now()); + + assert_eq!(1, summary.buckets().len()); + assert_eq!(2, summary.count()); + assert!(!summary.is_empty()); + } +} diff --git a/metrics-exporter-prometheus/src/formatting.rs b/metrics-exporter-prometheus/src/formatting.rs index b0fd4b78..75f61524 100644 --- a/metrics-exporter-prometheus/src/formatting.rs +++ b/metrics-exporter-prometheus/src/formatting.rs @@ -15,7 +15,7 @@ pub fn key_to_parts( ) -> (String, Vec) { let name = sanitize_metric_name(key.name()); let mut values = default_labels.cloned().unwrap_or_default(); - key.labels().into_iter().for_each(|label| { + key.labels().for_each(|label| { values.insert(label.key().to_string(), label.value().to_string()); }); let labels = values @@ -128,9 +128,17 @@ pub fn sanitize_metric_name(name: &str) -> String { /// [data model]: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels pub fn sanitize_label_key(key: &str) -> String { // The first character must be [a-zA-Z_], and all subsequent characters must be [a-zA-Z0-9_]. - key.replacen(invalid_label_key_start_character, "_", 1) - .replace(invalid_label_key_character, "_") - .replacen("__", "___", 1) + let mut out = String::with_capacity(key.len()); + let mut is_invalid: fn(char) -> bool = invalid_label_key_start_character; + for c in key.chars() { + if is_invalid(c) { + out.push('_'); + } else { + out.push(c); + } + is_invalid = invalid_label_key_character; + } + out } /// Sanitizes a label value to be valid under the Prometheus [data model]. @@ -241,6 +249,8 @@ mod tests { ("foo_bar", "foo_bar"), ("foo1_bar", "foo1_bar"), ("1foobar", "_foobar"), + ("foo1:bar2", "foo1:bar2"), + ("123", "_23"), ]; for (input, expected) in cases { @@ -257,7 +267,9 @@ mod tests { (":", "_"), ("foo_bar", "foo_bar"), ("1foobar", "_foobar"), - ("__foobar", "___foobar"), + ("__foobar", "__foobar"), + ("foo1bar2", "foo1bar2"), + ("123", "_23"), ]; for (input, expected) in cases { @@ -329,13 +341,17 @@ mod tests { // Label keys cannot begin with two underscores, as that format is reserved for internal // use. - if as_chars.len() == 2 { + // + // TODO: More closely examine how official Prometheus client libraries handle label key sanitization + // and follow whatever they do, so it's not actually clear if transforming `__foo` to `___foo` would + // be valid, given that it still technically starts with two underscores. + /*if as_chars.len() == 2 { assert!(!(as_chars[0] == '_' && as_chars[1] == '_')); } else if as_chars.len() == 3 { if as_chars[0] == '_' && as_chars[1] == '_' { assert_eq!(as_chars[2], '_'); } - } + }*/ assert!(!as_chars.iter().any(|c| invalid_label_key_character(*c)), "invalid character in label key"); diff --git a/metrics-exporter-prometheus/src/lib.rs b/metrics-exporter-prometheus/src/lib.rs index 8f6de32c..399d942b 100644 --- a/metrics-exporter-prometheus/src/lib.rs +++ b/metrics-exporter-prometheus/src/lib.rs @@ -112,4 +112,6 @@ pub use self::builder::PrometheusBuilder; pub mod formatting; mod recorder; +mod registry; + pub use self::recorder::{PrometheusHandle, PrometheusRecorder}; diff --git a/metrics-exporter-prometheus/src/recorder.rs b/metrics-exporter-prometheus/src/recorder.rs index ee94ec84..d252eb41 100644 --- a/metrics-exporter-prometheus/src/recorder.rs +++ b/metrics-exporter-prometheus/src/recorder.rs @@ -2,24 +2,26 @@ use std::collections::HashMap; use std::error::Error; use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::{PoisonError, RwLock}; use indexmap::IndexMap; -use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; -use metrics_util::registry::{GenerationalAtomicStorage, Recency, Registry}; -use parking_lot::RwLock; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; +use metrics_util::registry::{Recency, Registry}; +use quanta::Instant; use crate::common::Snapshot; use crate::distribution::{Distribution, DistributionBuilder}; use crate::formatting::{ key_to_parts, sanitize_metric_name, write_help_line, write_metric_line, write_type_line, }; +use crate::registry::GenerationalAtomicStorage; pub(crate) struct Inner { pub registry: Registry, pub recency: Recency, pub distributions: RwLock, Distribution>>>, pub distribution_builder: DistributionBuilder, - pub descriptions: RwLock>, + pub descriptions: RwLock>, pub global_labels: IndexMap, } @@ -99,7 +101,7 @@ impl Inner { // is not recent enough and should be/was deleted from the registry, we also need to // delete it on our side as well. let (name, labels) = key_to_parts(&key, Some(&self.global_labels)); - let mut wg = self.distributions.write(); + let mut wg = self.distributions.write().unwrap_or_else(PoisonError::into_inner); let delete_by_name = if let Some(by_name) = wg.get_mut(&name) { by_name.remove(&labels); by_name.is_empty() @@ -118,17 +120,18 @@ impl Inner { let (name, labels) = key_to_parts(&key, Some(&self.global_labels)); - let mut wg = self.distributions.write(); + let mut wg = self.distributions.write().unwrap_or_else(PoisonError::into_inner); let entry = wg .entry(name.clone()) - .or_insert_with(IndexMap::new) + .or_default() .entry(labels) .or_insert_with(|| self.distribution_builder.get_distribution(name.as_str())); histogram.get_inner().clear_with(|samples| entry.record_samples(samples)); } - let distributions = self.distributions.read().clone(); + let distributions = + self.distributions.read().unwrap_or_else(PoisonError::into_inner).clone(); Snapshot { counters, gauges, distributions } } @@ -155,6 +158,7 @@ impl Inner { for (labels, distribution) in by_labels.drain(..) { let (sum, count) = match distribution { Distribution::Summary(summary, quantiles, sum) => { + let summary = summary.snapshot(Instant::now()); for quantile in quantiles.iter() { let value = summary.quantile(quantile.value()).unwrap_or(0.0); let additional = Some(format!("quantile={}", quantile.value())); @@ -195,7 +199,7 @@ impl Inner { let Snapshot { mut counters, mut distributions, mut gauges } = self.get_recent_metrics(); let mut output = String::new(); - let descriptions = self.descriptions.read(); + let descriptions = self.descriptions.read().unwrap_or_else(PoisonError::into_inner); for (name, mut by_labels) in counters.drain() { if let Some(desc) = descriptions.get(name.as_str()) { @@ -231,8 +235,9 @@ impl Inner { for (labels, distribution) in by_labels.drain(..) { let (sum, count) = match distribution { Distribution::Summary(summary, quantiles, sum) => { + let snapshot = summary.snapshot(Instant::now()); for quantile in quantiles.iter() { - let value = summary.quantile(quantile.value()).unwrap_or(0.0); + let value = snapshot.quantile(quantile.value()).unwrap_or(0.0); write_metric_line( &mut output, &name, @@ -288,7 +293,10 @@ impl Inner { pub fn clear(&self) { self.registry.clear(); - self.distributions.write().clear(); + let lock = self.distributions.write(); + if let Ok(mut lock) = lock { + lock.clear(); + } } } @@ -310,9 +318,10 @@ impl PrometheusRecorder { PrometheusHandle { inner: self.inner.clone() } } - fn add_description_if_missing(&self, key_name: &KeyName, description: &'static str) { + fn add_description_if_missing(&self, key_name: &KeyName, description: SharedString) { let sanitized = sanitize_metric_name(key_name.as_str()); - let mut descriptions = self.inner.descriptions.write(); + let mut descriptions = + self.inner.descriptions.write().unwrap_or_else(PoisonError::into_inner); descriptions.entry(sanitized).or_insert(description); } } @@ -324,11 +333,11 @@ impl From for PrometheusRecorder { } impl Recorder for PrometheusRecorder { - fn describe_counter(&self, key_name: KeyName, _unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, _unit: Option, description: SharedString) { self.add_description_if_missing(&key_name, description); } - fn describe_gauge(&self, key_name: KeyName, _unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, _unit: Option, description: SharedString) { self.add_description_if_missing(&key_name, description); } @@ -336,20 +345,20 @@ impl Recorder for PrometheusRecorder { &self, key_name: KeyName, _unit: Option, - description: &'static str, + description: SharedString, ) { self.add_description_if_missing(&key_name, description); } - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { self.inner.registry.get_or_create_counter(key, |c| c.clone().into()) } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { self.inner.registry.get_or_create_gauge(key, |c| c.clone().into()) } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { self.inner.registry.get_or_create_histogram(key, |c| c.clone().into()) } } diff --git a/metrics-exporter-prometheus/src/registry.rs b/metrics-exporter-prometheus/src/registry.rs new file mode 100644 index 00000000..c6001743 --- /dev/null +++ b/metrics-exporter-prometheus/src/registry.rs @@ -0,0 +1,53 @@ +use std::sync::Arc; + +use metrics::{atomics::AtomicU64, HistogramFn}; +use metrics_util::{registry::GenerationalStorage, AtomicBucket}; +use quanta::Instant; + +pub type GenerationalAtomicStorage = GenerationalStorage; + +/// Atomic metric storage for the prometheus exporter. +pub struct AtomicStorage; + +impl metrics_util::registry::Storage for AtomicStorage { + type Counter = Arc; + type Gauge = Arc; + type Histogram = Arc>; + + fn counter(&self, _: &K) -> Self::Counter { + Arc::new(AtomicU64::new(0)) + } + + fn gauge(&self, _: &K) -> Self::Gauge { + Arc::new(AtomicU64::new(0)) + } + + fn histogram(&self, _: &K) -> Self::Histogram { + Arc::new(AtomicBucketInstant::new()) + } +} + +/// An `AtomicBucket` newtype wrapper that tracks the time of value insertion. +pub struct AtomicBucketInstant { + inner: AtomicBucket<(T, Instant)>, +} + +impl AtomicBucketInstant { + fn new() -> AtomicBucketInstant { + Self { inner: AtomicBucket::new() } + } + + pub fn clear_with(&self, f: F) + where + F: FnMut(&[(T, Instant)]), + { + self.inner.clear_with(f); + } +} + +impl HistogramFn for AtomicBucketInstant { + fn record(&self, value: f64) { + let now = Instant::now(); + self.inner.push((value, now)); + } +} diff --git a/metrics-exporter-tcp/CHANGELOG.md b/metrics-exporter-tcp/CHANGELOG.md index d22baf26..ce05c71e 100644 --- a/metrics-exporter-tcp/CHANGELOG.md +++ b/metrics-exporter-tcp/CHANGELOG.md @@ -8,6 +8,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - ReleaseDate +## [0.8.0] - 2023-04-16 + +### Changed + +- Bump MSRV to 1.61.0. +- Implemented `std::error::Error` for `Error`. + +## [0.7.0] - 2022-07-20 + +### Changed + +- Update `metrics` to `0.20`. + ## [0.6.0] - 2022-01-14 ### Changed diff --git a/metrics-exporter-tcp/Cargo.toml b/metrics-exporter-tcp/Cargo.toml index 9f69c932..7801ea3c 100644 --- a/metrics-exporter-tcp/Cargo.toml +++ b/metrics-exporter-tcp/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "metrics-exporter-tcp" -version = "0.6.0" +version = "0.8.0" authors = ["Toby Lawrence "] edition = "2018" +rust-version = "1.61.0" license = "MIT" @@ -16,19 +17,19 @@ categories = ["development-tools::debugging"] keywords = ["metrics", "telemetry", "tcp"] [dependencies] -metrics = { version = "^0.18", path = "../metrics" } +metrics = { version = "^0.21", path = "../metrics" } bytes = { version = "1", default-features = false } crossbeam-channel = { version = "0.5", default-features = false, features = ["std"] } -prost = { version = "0.9", default-features = false } -prost-types = { version = "0.9", default-features = false, features = ["std"] } +prost = { version = "0.11", default-features = false } +prost-types = { version = "0.11", default-features = false, features = ["std"] } mio = { version = "0.8", default-features = false, features = ["os-poll", "net"] } tracing = { version = "0.1", default-features = false, features = ["attributes"] } [build-dependencies] -prost-build = "0.9" +prost-build = "0.11" [dev-dependencies] -quanta = "0.9.3" +quanta = "0.12" tracing = "0.1" tracing-subscriber = "0.3" rand = "0.8" diff --git a/metrics-exporter-tcp/build.rs b/metrics-exporter-tcp/build.rs index 15559df8..7fe5a9ef 100644 --- a/metrics-exporter-tcp/build.rs +++ b/metrics-exporter-tcp/build.rs @@ -1,6 +1,6 @@ fn main() { println!("cargo:rerun-if-changed=proto/event.proto"); let mut prost_build = prost_build::Config::new(); - prost_build.btree_map(&["."]); + prost_build.btree_map(["."]); prost_build.compile_protos(&["proto/event.proto"], &["proto/"]).unwrap(); } diff --git a/metrics-exporter-tcp/examples/tcp_server.rs b/metrics-exporter-tcp/examples/tcp_server.rs index 315777b3..1953fa2b 100644 --- a/metrics-exporter-tcp/examples/tcp_server.rs +++ b/metrics-exporter-tcp/examples/tcp_server.rs @@ -1,9 +1,7 @@ use std::thread; use std::time::Duration; -use metrics::{ - decrement_gauge, describe_histogram, histogram, increment_counter, increment_gauge, Unit, -}; +use metrics::{counter, describe_histogram, gauge, histogram, Unit}; use metrics_exporter_tcp::TcpBuilder; use quanta::Clock; @@ -25,18 +23,19 @@ fn main() { ); loop { - increment_counter!("tcp_server_loops", "system" => "foo"); + counter!("tcp_server_loops", "system" => "foo").increment(1); if let Some(t) = last { let delta: Duration = clock.now() - t; - histogram!("tcp_server_loop_delta_secs", delta, "system" => "foo"); + histogram!("tcp_server_loop_delta_secs", "system" => "foo").record(delta); } let increment_gauge = thread_rng().gen_bool(0.75); + let gauge = gauge!("lucky_iterations"); if increment_gauge { - increment_gauge!("lucky_iterations", 1.0); + gauge.increment(1.0); } else { - decrement_gauge!("lucky_iterations", 1.0); + gauge.decrement(1.0); } last = Some(clock.now()); diff --git a/metrics-exporter-tcp/src/lib.rs b/metrics-exporter-tcp/src/lib.rs index 1c35a11f..0c4fc7c8 100644 --- a/metrics-exporter-tcp/src/lib.rs +++ b/metrics-exporter-tcp/src/lib.rs @@ -62,8 +62,8 @@ use std::{ use bytes::Bytes; use crossbeam_channel::{bounded, unbounded, Receiver, Sender}; use metrics::{ - Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Recorder, - SetRecorderError, Unit, + Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, Recorder, + SetRecorderError, SharedString, Unit, }; use mio::{ net::{TcpListener, TcpStream}, @@ -93,7 +93,7 @@ enum MetricOperation { } enum Event { - Metadata(KeyName, MetricType, Option, &'static str), + Metadata(KeyName, MetricType, Option, SharedString), Metric(Key, MetricOperation), } @@ -119,6 +119,24 @@ impl From for Error { } } +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Io(e) => write!(f, "IO error: {}", e), + Error::Recorder(e) => write!(f, "recorder error: {}", e), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::Io(e) => Some(e), + Error::Recorder(e) => Some(e), + } + } +} + struct State { client_count: AtomicUsize, should_send: AtomicBool, @@ -152,7 +170,7 @@ impl State { key_name: KeyName, metric_type: MetricType, unit: Option, - description: &'static str, + description: SharedString, ) { let _ = self.tx.try_send(Event::Metadata(key_name, metric_type, unit, description)); self.wake(); @@ -303,27 +321,27 @@ impl Default for TcpBuilder { } impl Recorder for TcpRecorder { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString) { self.state.register_metric(key_name, MetricType::Counter, unit, description); } - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { self.state.register_metric(key_name, MetricType::Gauge, unit, description); } - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString) { self.state.register_metric(key_name, MetricType::Histogram, unit, description); } - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { Counter::from_arc(Arc::new(Handle::new(key.clone(), self.state.clone()))) } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { Gauge::from_arc(Arc::new(Handle::new(key.clone(), self.state.clone()))) } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { Histogram::from_arc(Arc::new(Handle::new(key.clone(), self.state.clone()))) } } @@ -501,12 +519,13 @@ fn run_transport( #[allow(clippy::mutable_key_type)] fn generate_metadata_messages( - metadata: &HashMap, Option<&'static str>)>, + metadata: &HashMap, Option)>, ) -> VecDeque { let mut bufs = VecDeque::new(); for (key_name, (metric_type, unit, desc)) in metadata.iter() { - let msg = convert_metadata_to_protobuf_encoded(key_name, *metric_type, *unit, *desc) - .expect("failed to encode metadata buffer"); + let msg = + convert_metadata_to_protobuf_encoded(key_name, *metric_type, *unit, desc.as_ref()) + .expect("failed to encode metadata buffer"); bufs.push_back(msg); } bufs @@ -562,14 +581,14 @@ fn convert_metadata_to_protobuf_encoded( key_name: &KeyName, metric_type: MetricType, unit: Option, - desc: Option<&'static str>, + desc: Option<&SharedString>, ) -> Result { let name = key_name.as_str().to_string(); let metadata = proto::Metadata { name, metric_type: metric_type.into(), unit: unit.map(|u| proto::metadata::Unit::UnitValue(u.as_str().to_owned())), - description: desc.map(|d| proto::metadata::Description::DescriptionValue(d.to_owned())), + description: desc.map(|d| proto::metadata::Description::DescriptionValue(d.to_string())), }; let event = proto::Event { event: Some(proto::event::Event::Metadata(metadata)) }; diff --git a/metrics-macros/CHANGELOG.md b/metrics-macros/CHANGELOG.md deleted file mode 100644 index 856f20c1..00000000 --- a/metrics-macros/CHANGELOG.md +++ /dev/null @@ -1,58 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - - - -## [Unreleased] - ReleaseDate - -## [0.5.1] - 2022-02-06 - -## [0.5.0] - 2022-01-14 - -### Added -- When describing a metric, a constant can now be used for the description itself. -- Label keys can now be general expressions i.e. constants or variables. Due to limitations in - how procedural macros work, and the facilities available in stable Rust for const traits, even - `&'static str` constants will cause allocations when used for emitting a metric. - -### Changed -- Correctly scoped the required features of various dependencies to reduce build times/transitive dependencies. -- Updated macros to coincide with the update to `metrics` for metric handles. This includes - renaming `register_*` macros to `describe_*`, which are purely for providing data that describes a - metric but does not initialize it in any way, and providing new `register_*` macros which do - initialize a metric. -- Updated the `describe_*` macros -- née `register_*` -- to require a description, and an optional - unit. As describing a metric does not register it in the sense of ensuring that it is present on - the output of an exporter, having the description be optional no longer makes sense. -- Additionally, the describe macros no longer take labels. In practice, varying the description of - a specific metric based on label values would be counter-intuitive, and to support this corner - case requires adds significant complexity to the macro parsing logic. - -### Removed -- Two unecessary dependencies, `lazy_static` and `regex`. - -## [0.4.1] - 2021-12-16 - -### Changed -- Removed unnecessary `proc-macro-hack` dependency. - -## [0.4.0] - 2021-05-18 - -### Changed -- Updates to macros to support the removal of `NameParts` and related machinery. - -## [0.3.0] - 2021-05-03 - -### Changed -- Updates to macros to support changes in `Recorder` around how keys are taken. - -## [0.2.0] - 2021-02-02 -### Changed -- Added support for owned strings as metric names. [#170](https://github.com/metrics-rs/metrics/pull/170) - -## [0.1.0] - 2021-01-22 -### Added -- Effective birth of the crate. diff --git a/metrics-macros/Cargo.toml b/metrics-macros/Cargo.toml deleted file mode 100644 index 804ffac3..00000000 --- a/metrics-macros/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "metrics-macros" -version = "0.5.1" -authors = ["Toby Lawrence "] -edition = "2018" - -license = "MIT" - -description = "Macros for the metrics crate." -homepage = "https://github.com/metrics-rs/metrics" -repository = "https://github.com/metrics-rs/metrics" -documentation = "https://docs.rs/metrics" -readme = "README.md" - -categories = ["development-tools::debugging"] -keywords = ["metrics", "facade", "macros"] - -[lib] -proc-macro = true - -[dependencies] -syn = { version = "1", default-features = false, features = ["derive", "full", "parsing", "printing", "proc-macro"] } -quote = { version = "1", default-features = false } -proc-macro2 = { version = "1", default-features = false, features = ["proc-macro"] } diff --git a/metrics-macros/LICENSE b/metrics-macros/LICENSE deleted file mode 120000 index ea5b6064..00000000 --- a/metrics-macros/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../LICENSE \ No newline at end of file diff --git a/metrics-macros/README.md b/metrics-macros/README.md deleted file mode 100644 index bd6c1b7b..00000000 --- a/metrics-macros/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# metrics-macros - -This crate houses all of the procedural macros that are re-exported by `metrics`. Refer to the -documentation for `metrics` to find examples and more information on the available macros. \ No newline at end of file diff --git a/metrics-macros/src/lib.rs b/metrics-macros/src/lib.rs deleted file mode 100644 index 3d7ec5c0..00000000 --- a/metrics-macros/src/lib.rs +++ /dev/null @@ -1,470 +0,0 @@ -extern crate proc_macro; - -use self::proc_macro::TokenStream; - -use proc_macro2::TokenStream as TokenStream2; -use quote::{format_ident, quote, ToTokens}; -use syn::parse::{Error, Parse, ParseStream, Result}; -use syn::{parse::discouraged::Speculative, Lit}; -use syn::{parse_macro_input, Expr, Token}; - -#[cfg(test)] -mod tests; - -enum Labels { - Existing(Expr), - Inline(Vec<(Expr, Expr)>), -} - -struct WithoutExpression { - key: Expr, - labels: Option, -} - -struct WithExpression { - key: Expr, - op_value: Expr, - labels: Option, -} - -struct Description { - key: Expr, - unit: Option, - description: Expr, -} - -impl Parse for WithoutExpression { - fn parse(mut input: ParseStream) -> Result { - let key = input.parse::()?; - let labels = parse_labels(&mut input)?; - - Ok(WithoutExpression { key, labels }) - } -} - -impl Parse for WithExpression { - fn parse(mut input: ParseStream) -> Result { - let key = input.parse::()?; - - input.parse::()?; - let op_value = input.parse::()?; - - let labels = parse_labels(&mut input)?; - - Ok(WithExpression { key, op_value, labels }) - } -} - -impl Parse for Description { - fn parse(input: ParseStream) -> Result { - let key = input.parse::()?; - - // We accept two possible parameters: unit, and description. - // - // There is only one specific requirement that must be met, and that is that the unit _must_ - // have a qualified path of either `metrics::Unit::...` or `Unit::..` for us to properly - // distinguish it amongst the macro parameters. - - // Now try to read out the components. We speculatively try to parse out a unit if it - // exists, and otherwise we just look for the description. - let unit = input - .call(|s| { - let forked = s.fork(); - forked.parse::()?; - - let output = if let Ok(Expr::Path(path)) = forked.parse::() { - let qname = path - .path - .segments - .iter() - .map(|x| x.ident.to_string()) - .collect::>() - .join("::"); - if qname.starts_with("metrics::Unit") || qname.starts_with("Unit") { - Some(Expr::Path(path)) - } else { - None - } - } else { - None - }; - - if output.is_some() { - s.advance_to(&forked); - } - - Ok(output) - }) - .ok() - .flatten(); - - input.parse::()?; - let description = input.parse::()?; - - Ok(Description { key, unit, description }) - } -} - -#[proc_macro] -pub fn describe_counter(input: TokenStream) -> TokenStream { - let Description { key, unit, description } = parse_macro_input!(input as Description); - - get_describe_code("counter", key, unit, description).into() -} - -#[proc_macro] -pub fn describe_gauge(input: TokenStream) -> TokenStream { - let Description { key, unit, description } = parse_macro_input!(input as Description); - - get_describe_code("gauge", key, unit, description).into() -} - -#[proc_macro] -pub fn describe_histogram(input: TokenStream) -> TokenStream { - let Description { key, unit, description } = parse_macro_input!(input as Description); - - get_describe_code("histogram", key, unit, description).into() -} - -#[proc_macro] -pub fn register_counter(input: TokenStream) -> TokenStream { - let WithoutExpression { key, labels } = parse_macro_input!(input as WithoutExpression); - - get_register_and_op_code::("counter", key, labels, None).into() -} - -#[proc_macro] -pub fn register_gauge(input: TokenStream) -> TokenStream { - let WithoutExpression { key, labels } = parse_macro_input!(input as WithoutExpression); - - get_register_and_op_code::("gauge", key, labels, None).into() -} - -#[proc_macro] -pub fn register_histogram(input: TokenStream) -> TokenStream { - let WithoutExpression { key, labels } = parse_macro_input!(input as WithoutExpression); - - get_register_and_op_code::("histogram", key, labels, None).into() -} - -#[proc_macro] -pub fn increment_counter(input: TokenStream) -> TokenStream { - let WithoutExpression { key, labels } = parse_macro_input!(input as WithoutExpression); - - let op_value = quote! { 1 }; - - get_register_and_op_code("counter", key, labels, Some(("increment", op_value))).into() -} - -#[proc_macro] -pub fn counter(input: TokenStream) -> TokenStream { - let WithExpression { key, op_value, labels } = parse_macro_input!(input as WithExpression); - - get_register_and_op_code("counter", key, labels, Some(("increment", op_value))).into() -} - -#[proc_macro] -pub fn absolute_counter(input: TokenStream) -> TokenStream { - let WithExpression { key, op_value, labels } = parse_macro_input!(input as WithExpression); - - get_register_and_op_code("counter", key, labels, Some(("absolute", op_value))).into() -} - -#[proc_macro] -pub fn increment_gauge(input: TokenStream) -> TokenStream { - let WithExpression { key, op_value, labels } = parse_macro_input!(input as WithExpression); - - get_register_and_op_code("gauge", key, labels, Some(("increment", op_value))).into() -} - -#[proc_macro] -pub fn decrement_gauge(input: TokenStream) -> TokenStream { - let WithExpression { key, op_value, labels } = parse_macro_input!(input as WithExpression); - - get_register_and_op_code("gauge", key, labels, Some(("decrement", op_value))).into() -} - -#[proc_macro] -pub fn gauge(input: TokenStream) -> TokenStream { - let WithExpression { key, op_value, labels } = parse_macro_input!(input as WithExpression); - - get_register_and_op_code("gauge", key, labels, Some(("set", op_value))).into() -} - -#[proc_macro] -pub fn histogram(input: TokenStream) -> TokenStream { - let WithExpression { key, op_value, labels } = parse_macro_input!(input as WithExpression); - - get_register_and_op_code("histogram", key, labels, Some(("record", op_value))).into() -} - -fn get_describe_code( - metric_type: &str, - name: Expr, - unit: Option, - description: Expr, -) -> TokenStream2 { - let describe_ident = format_ident!("describe_{}", metric_type); - - let unit = match unit { - Some(e) => quote! { Some(#e) }, - None => quote! { None }, - }; - - quote! { - { - // Only do this work if there's a recorder installed. - if let Some(recorder) = metrics::try_recorder() { - recorder.#describe_ident(#name.into(), #unit, #description); - } - } - } -} - -fn get_register_and_op_code( - metric_type: &str, - name: Expr, - labels: Option, - op: Option<(&'static str, V)>, -) -> TokenStream2 -where - V: ToTokens, -{ - let register_ident = format_ident!("register_{}", metric_type); - let statics = generate_statics(&name, &labels); - let (locals, metric_key) = generate_metric_key(&name, &labels); - match op { - Some((op_type, op_value)) => { - let op_ident = format_ident!("{}", op_type); - let op_value = if metric_type == "histogram" { - quote! { metrics::__into_f64(#op_value) } - } else { - quote! { #op_value } - }; - - // We've been given values to actually use with the handle, so we actually check if a - // recorder is installed before bothering to create a handle and everything. - quote! { - { - #statics - // Only do this work if there's a recorder installed. - if let Some(recorder) = metrics::try_recorder() { - #locals - let handle = recorder.#register_ident(#metric_key); - handle.#op_ident(#op_value); - } - } - } - } - None => { - // If there's no values specified, we simply return the metric handle. - quote! { - { - #statics - #locals - metrics::recorder().#register_ident(#metric_key) - } - } - } - } -} - -fn name_is_fast_path(name: &Expr) -> bool { - if let Expr::Lit(lit) = name { - return matches!(lit.lit, Lit::Str(_)); - } - - false -} - -fn labels_are_fast_path(labels: &Labels) -> bool { - match labels { - Labels::Existing(_) => false, - Labels::Inline(pairs) => { - pairs.iter().all(|(k, v)| matches!((k, v), (Expr::Lit(_), Expr::Lit(_)))) - } - } -} - -fn generate_statics(name: &Expr, labels: &Option) -> TokenStream2 { - // Create the static for the name, if possible. - let use_name_static = name_is_fast_path(name); - let name_static = if use_name_static { - quote! { - static METRIC_NAME: &'static str = #name; - } - } else { - quote! {} - }; - - // Create the static for the labels, if possible. - let has_labels = labels.is_some(); - let use_labels_static = match labels.as_ref() { - Some(labels) => labels_are_fast_path(labels), - None => true, - }; - - let labels_static = match labels.as_ref() { - Some(labels) => { - if labels_are_fast_path(labels) { - if let Labels::Inline(pairs) = labels { - let labels = pairs - .iter() - .map(|(key, val)| quote! { metrics::Label::from_static_parts(#key, #val) }) - .collect::>(); - let labels_len = labels.len(); - let labels_len = quote! { #labels_len }; - - quote! { - static METRIC_LABELS: [metrics::Label; #labels_len] = [#(#labels),*]; - } - } else { - quote! {} - } - } else { - quote! {} - } - } - None => quote! {}, - }; - - let key_static = if use_name_static && use_labels_static { - if has_labels { - quote! { - static METRIC_KEY: metrics::Key = metrics::Key::from_static_parts(METRIC_NAME, &METRIC_LABELS); - } - } else { - quote! { - static METRIC_KEY: metrics::Key = metrics::Key::from_static_name(METRIC_NAME); - } - } - } else { - quote! {} - }; - - quote! { - #name_static - #labels_static - #key_static - } -} - -fn generate_metric_key(name: &Expr, labels: &Option) -> (TokenStream2, TokenStream2) { - let use_name_static = name_is_fast_path(name); - - let has_labels = labels.is_some(); - let use_labels_static = match labels.as_ref() { - Some(labels) => labels_are_fast_path(labels), - None => true, - }; - - let mut key_name = quote! { &key }; - let locals = if use_name_static && use_labels_static { - // Key is entirely static, so we can simply reference our generated statics. They will be - // inclusive of whether or not labels were specified. - key_name = quote! { &METRIC_KEY }; - quote! {} - } else if use_name_static && !use_labels_static { - // The name is static, but we have labels which are not static. Since `use_labels_static` - // cannot be false unless labels _are_ specified, we know this unwrap is safe. - let labels = labels.as_ref().unwrap(); - let quoted_labels = labels_to_quoted(labels); - quote! { - let key = metrics::Key::from_parts(METRIC_NAME, #quoted_labels); - } - } else if !use_name_static && !use_labels_static { - // The name is not static, and neither are the labels. Since `use_labels_static` - // cannot be false unless labels _are_ specified, we know this unwrap is safe. - let labels = labels.as_ref().unwrap(); - let quoted_labels = labels_to_quoted(labels); - quote! { - let key = metrics::Key::from_parts(#name, #quoted_labels); - } - } else { - // The name is not static, but the labels are. This could technically mean that there - // simply are no labels, so we have to discriminate in a slightly different way - // to figure out the correct key. - if has_labels { - quote! { - let key = metrics::Key::from_static_labels(#name, &METRIC_LABELS); - } - } else { - quote! { - let key = metrics::Key::from_name(#name); - } - } - }; - - (locals, key_name) -} - -fn labels_to_quoted(labels: &Labels) -> proc_macro2::TokenStream { - match labels { - Labels::Inline(pairs) => { - let labels = pairs.iter().map(|(key, val)| quote! { metrics::Label::new(#key, #val) }); - quote! { vec![#(#labels),*] } - } - Labels::Existing(e) => quote! { #e }, - } -} - -fn parse_labels(input: &mut ParseStream) -> Result> { - if input.is_empty() { - return Ok(None); - } - - if !input.peek(Token![,]) { - // This is a hack to generate the proper error message for parsing the comma next without - // actually parsing it and thus removing it from the parse stream. Just makes the following - // code a bit cleaner. - input - .parse::() - .map_err(|e| Error::new(e.span(), "expected labels, but comma not found"))?; - } - - // Two possible states for labels: references to a label iterator, or key/value pairs. - // - // We check to see if we have the ", key =>" part, which tells us that we're taking in key/value - // pairs. If we don't have that, we check to see if we have a "`, ]) { - let mut labels = Vec::new(); - loop { - if input.is_empty() { - break; - } - input.parse::()?; - if input.is_empty() { - break; - } - - let k = input.parse::()?; - input.parse::]>()?; - let v = input.parse::()?; - - labels.push((k, v)); - } - - return Ok(Some(Labels::Inline(labels))); - } - - // Has to be an expression otherwise, or a trailing comma. - input.parse::()?; - - // Unless it was an expression - clear the trailing comma. - if input.is_empty() { - return Ok(None); - } - - let existing = input.parse::().map_err(|e| { - Error::new(e.span(), "expected labels expression, but expression not found") - })?; - - // Expression can end with a trailing comma, handle it. - if input.peek(Token![,]) { - input.parse::()?; - } - - Ok(Some(Labels::Existing(existing))) -} diff --git a/metrics-macros/src/tests.rs b/metrics-macros/src/tests.rs deleted file mode 100644 index 34f9be57..00000000 --- a/metrics-macros/src/tests.rs +++ /dev/null @@ -1,523 +0,0 @@ -use syn::parse_quote; -use syn::{Expr, ExprPath}; - -use super::*; - -#[test] -fn test_get_describe_code() { - // Basic registration. - let stream = get_describe_code( - "mytype", - parse_quote! { "mykeyname" }, - None, - parse_quote! { "a counter" }, - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "recorder . describe_mytype (\"mykeyname\" . into () , None , \"a counter\") ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_describe_code_with_qualified_unit() { - // Now with unit. - let units: ExprPath = parse_quote! { metrics::Unit::Nanoseconds }; - let stream = get_describe_code( - "mytype", - parse_quote! { "mykeyname" }, - Some(Expr::Path(units)), - parse_quote! { "a counter" }, - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "recorder . describe_mytype (\"mykeyname\" . into () , Some (metrics :: Unit :: Nanoseconds) , \"a counter\") ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_describe_code_with_relative_unit() { - // Now with unit. - let units: ExprPath = parse_quote! { Unit::Nanoseconds }; - let stream = get_describe_code( - "mytype", - parse_quote! { "mykeyname" }, - Some(Expr::Path(units)), - parse_quote! { "a counter" }, - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "recorder . describe_mytype (\"mykeyname\" . into () , Some (Unit :: Nanoseconds) , \"a counter\") ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_describe_code_with_constants() { - // Basic registration. - let stream = - get_describe_code("mytype", parse_quote! { KEY_NAME }, None, parse_quote! { COUNTER_DESC }); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "recorder . describe_mytype (KEY_NAME . into () , None , COUNTER_DESC) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_describe_code_with_constants_and_with_qualified_unit() { - // Now with unit. - let units: ExprPath = parse_quote! { metrics::Unit::Nanoseconds }; - let stream = get_describe_code( - "mytype", - parse_quote! { KEY_NAME }, - Some(Expr::Path(units)), - parse_quote! { COUNTER_DESC }, - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "recorder . describe_mytype (KEY_NAME . into () , Some (metrics :: Unit :: Nanoseconds) , COUNTER_DESC) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_describe_code_with_constants_and_with_relative_unit() { - // Now with unit. - let units: ExprPath = parse_quote! { Unit::Nanoseconds }; - let stream = get_describe_code( - "mytype", - parse_quote! { KEY_NAME }, - Some(Expr::Path(units)), - parse_quote! { COUNTER_DESC }, - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "recorder . describe_mytype (KEY_NAME . into () , Some (Unit :: Nanoseconds) , COUNTER_DESC) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_register_static_name_no_labels() { - let stream = get_register_and_op_code::("mytype", parse_quote! {"mykeyname"}, None, None); - - let expected = concat!( - "{ ", - "static METRIC_NAME : & 'static str = \"mykeyname\" ; ", - "static METRIC_KEY : metrics :: Key = metrics :: Key :: from_static_name (METRIC_NAME) ; ", - "metrics :: recorder () . register_mytype (& METRIC_KEY) ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_register_static_name_static_inline_labels() { - let labels = Labels::Inline(vec![(parse_quote! { "key1" }, parse_quote! { "value1" })]); - let stream = - get_register_and_op_code::("mytype", parse_quote! {"mykeyname"}, Some(labels), None); - - let expected = concat!( - "{ ", - "static METRIC_NAME : & 'static str = \"mykeyname\" ; ", - "static METRIC_LABELS : [metrics :: Label ; 1usize] = [metrics :: Label :: from_static_parts (\"key1\" , \"value1\")] ; ", - "static METRIC_KEY : metrics :: Key = metrics :: Key :: from_static_parts (METRIC_NAME , & METRIC_LABELS) ; ", - "metrics :: recorder () . register_mytype (& METRIC_KEY) ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_register_static_name_dynamic_inline_labels() { - let labels = Labels::Inline(vec![(parse_quote! { "key1" }, parse_quote! { &value1 })]); - let stream = - get_register_and_op_code::("mytype", parse_quote! {"mykeyname"}, Some(labels), None); - - let expected = concat!( - "{ ", - "static METRIC_NAME : & 'static str = \"mykeyname\" ; ", - "let key = metrics :: Key :: from_parts (METRIC_NAME , vec ! [metrics :: Label :: new (\"key1\" , & value1)]) ; ", - "metrics :: recorder () . register_mytype (& key) ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -/// If there are dynamic labels - generate a direct invocation. -#[test] -fn test_get_register_and_op_code_register_static_name_existing_labels() { - let stream = get_register_and_op_code::( - "mytype", - parse_quote! {"mykeyname"}, - Some(Labels::Existing(parse_quote! { mylabels })), - None, - ); - - let expected = concat!( - "{ ", - "static METRIC_NAME : & 'static str = \"mykeyname\" ; ", - "let key = metrics :: Key :: from_parts (METRIC_NAME , mylabels) ; ", - "metrics :: recorder () . register_mytype (& key) ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_register_owned_name_no_labels() { - let stream = get_register_and_op_code::( - "mytype", - parse_quote! { String::from("owned") }, - None, - None, - ); - - let expected = concat!( - "{ ", - "let key = metrics :: Key :: from_name (String :: from (\"owned\")) ; ", - "metrics :: recorder () . register_mytype (& key) ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_register_owned_name_static_inline_labels() { - let labels = Labels::Inline(vec![(parse_quote! { "key1" }, parse_quote! { "value1" })]); - let stream = get_register_and_op_code::( - "mytype", - parse_quote! { String::from("owned") }, - Some(labels), - None, - ); - - let expected = concat!( - "{ ", - "static METRIC_LABELS : [metrics :: Label ; 1usize] = [metrics :: Label :: from_static_parts (\"key1\" , \"value1\")] ; ", - "let key = metrics :: Key :: from_static_labels (String :: from (\"owned\") , & METRIC_LABELS) ; ", - "metrics :: recorder () . register_mytype (& key) ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_register_owned_name_dynamic_inline_labels() { - let labels = Labels::Inline(vec![(parse_quote! { "key1" }, parse_quote! { &value1 })]); - let stream = get_register_and_op_code::( - "mytype", - parse_quote! { String::from("owned") }, - Some(labels), - None, - ); - - let expected = concat!( - "{ ", - "let key = metrics :: Key :: from_parts (String :: from (\"owned\") , vec ! [metrics :: Label :: new (\"key1\" , & value1)]) ; ", - "metrics :: recorder () . register_mytype (& key) ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -/// If there are dynamic labels - generate a direct invocation. -#[test] -fn test_get_register_and_op_code_register_owned_name_existing_labels() { - let stream = get_register_and_op_code::( - "mytype", - parse_quote! { String::from("owned") }, - Some(Labels::Existing(parse_quote! { mylabels })), - None, - ); - - let expected = concat!( - "{ ", - "let key = metrics :: Key :: from_parts (String :: from (\"owned\") , mylabels) ; ", - "metrics :: recorder () . register_mytype (& key) ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_op_static_name_no_labels() { - let stream = get_register_and_op_code( - "mytype", - parse_quote! {"mykeyname"}, - None, - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "static METRIC_NAME : & 'static str = \"mykeyname\" ; ", - "static METRIC_KEY : metrics :: Key = metrics :: Key :: from_static_name (METRIC_NAME) ; ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let handle = recorder . register_mytype (& METRIC_KEY) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_op_static_name_static_inline_labels() { - let labels = Labels::Inline(vec![(parse_quote! { "key1" }, parse_quote! { "value1" })]); - let stream = get_register_and_op_code( - "mytype", - parse_quote! {"mykeyname"}, - Some(labels), - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "static METRIC_NAME : & 'static str = \"mykeyname\" ; ", - "static METRIC_LABELS : [metrics :: Label ; 1usize] = [metrics :: Label :: from_static_parts (\"key1\" , \"value1\")] ; ", - "static METRIC_KEY : metrics :: Key = metrics :: Key :: from_static_parts (METRIC_NAME , & METRIC_LABELS) ; ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let handle = recorder . register_mytype (& METRIC_KEY) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_op_static_name_dynamic_inline_labels() { - let labels = Labels::Inline(vec![(parse_quote! { "key1" }, parse_quote! { &value1 })]); - let stream = get_register_and_op_code( - "mytype", - parse_quote! {"mykeyname"}, - Some(labels), - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "static METRIC_NAME : & 'static str = \"mykeyname\" ; ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let key = metrics :: Key :: from_parts (METRIC_NAME , vec ! [metrics :: Label :: new (\"key1\" , & value1)]) ; ", - "let handle = recorder . register_mytype (& key) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -/// If there are dynamic labels - generate a direct invocation. -#[test] -fn test_get_register_and_op_code_op_static_name_existing_labels() { - let stream = get_register_and_op_code( - "mytype", - parse_quote! {"mykeyname"}, - Some(Labels::Existing(parse_quote! { mylabels })), - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "static METRIC_NAME : & 'static str = \"mykeyname\" ; ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let key = metrics :: Key :: from_parts (METRIC_NAME , mylabels) ; ", - "let handle = recorder . register_mytype (& key) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_op_owned_name_no_labels() { - let stream = get_register_and_op_code( - "mytype", - parse_quote! { String::from("owned") }, - None, - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let key = metrics :: Key :: from_name (String :: from (\"owned\")) ; ", - "let handle = recorder . register_mytype (& key) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_op_owned_name_static_inline_labels() { - let labels = Labels::Inline(vec![(parse_quote! { "key1" }, parse_quote! { "value1" })]); - let stream = get_register_and_op_code( - "mytype", - parse_quote! { String::from("owned") }, - Some(labels), - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "static METRIC_LABELS : [metrics :: Label ; 1usize] = [metrics :: Label :: from_static_parts (\"key1\" , \"value1\")] ; ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let key = metrics :: Key :: from_static_labels (String :: from (\"owned\") , & METRIC_LABELS) ; ", - "let handle = recorder . register_mytype (& key) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_op_owned_name_dynamic_inline_labels() { - let labels = Labels::Inline(vec![(parse_quote! { "key1" }, parse_quote! { &value1 })]); - let stream = get_register_and_op_code( - "mytype", - parse_quote! { String::from("owned") }, - Some(labels), - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let key = metrics :: Key :: from_parts (String :: from (\"owned\") , vec ! [metrics :: Label :: new (\"key1\" , & value1)]) ; ", - "let handle = recorder . register_mytype (& key) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -/// If there are dynamic labels - generate a direct invocation. -#[test] -fn test_get_register_and_op_code_op_owned_name_existing_labels() { - let stream = get_register_and_op_code( - "mytype", - parse_quote! { String::from("owned") }, - Some(Labels::Existing(parse_quote! { mylabels })), - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let key = metrics :: Key :: from_parts (String :: from (\"owned\") , mylabels) ; ", - "let handle = recorder . register_mytype (& key) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_get_register_and_op_code_op_owned_name_constant_key_labels() { - let stream = get_register_and_op_code( - "mytype", - parse_quote! { String::from("owned") }, - Some(Labels::Inline(vec![(parse_quote! { LABEL_KEY }, parse_quote! { "some_val" })])), - Some(("myop", quote! { 1 })), - ); - - let expected = concat!( - "{ ", - "if let Some (recorder) = metrics :: try_recorder () { ", - "let key = metrics :: Key :: from_parts (String :: from (\"owned\") , vec ! [metrics :: Label :: new (LABEL_KEY , \"some_val\")]) ; ", - "let handle = recorder . register_mytype (& key) ; ", - "handle . myop (1) ; ", - "} ", - "}", - ); - - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_labels_to_quoted_existing_labels() { - let labels = Labels::Existing(Expr::Path(parse_quote! { mylabels })); - let stream = labels_to_quoted(&labels); - let expected = "mylabels"; - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_labels_to_quoted_inline_labels() { - let labels = Labels::Inline(vec![ - (parse_quote! {"mylabel1"}, parse_quote! { mylabel1 }), - (parse_quote! {"mylabel2"}, parse_quote! { "mylabel2" }), - ]); - let stream = labels_to_quoted(&labels); - let expected = concat!( - "vec ! [", - "metrics :: Label :: new (\"mylabel1\" , mylabel1) , ", - "metrics :: Label :: new (\"mylabel2\" , \"mylabel2\")", - "]" - ); - assert_eq!(stream.to_string(), expected); -} - -#[test] -fn test_labels_to_quoted_inline_labels_empty() { - let labels = Labels::Inline(vec![]); - let stream = labels_to_quoted(&labels); - let expected = "vec ! []"; - assert_eq!(stream.to_string(), expected); -} diff --git a/metrics-observer/CHANGELOG.md b/metrics-observer/CHANGELOG.md index 9b760239..2d3f581e 100644 --- a/metrics-observer/CHANGELOG.md +++ b/metrics-observer/CHANGELOG.md @@ -7,9 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - ReleaseDate + +### Fixed + +- All addresses returned when trying to connect to the specified exporter endpoint will be tried, in + order, instead of only trying the first and then giving up. + ([#429](https://github.com/metrics-rs/metrics/pull/429)) + +## [0.2.0] - 2023-04-16 + ### Added -- Update hdrhistogram dependency to 7.2 + +- Update `hdrhistogram`` dependency to 7.2 ### Changed + +- Bump MSRV to 1.61.0. - Updated various dependencies in order to properly scope dependencies to only the necessary feature flags, and thus optimize build times and reduce transitive dependencies. diff --git a/metrics-observer/Cargo.toml b/metrics-observer/Cargo.toml index 4151a99a..9a50cd21 100644 --- a/metrics-observer/Cargo.toml +++ b/metrics-observer/Cargo.toml @@ -1,22 +1,31 @@ [package] name = "metrics-observer" -version = "0.1.1-alpha.2" +version = "0.2.0" authors = ["Toby Lawrence "] edition = "2018" -publish = false +rust-version = "1.61.0" license = "MIT" +description = "Text-based UI for metrics-exporter-tcp." +homepage = "https://github.com/metrics-rs/metrics" +repository = "https://github.com/metrics-rs/metrics" +documentation = "https://docs.rs/metrics" +readme = "README.md" + +categories = ["development-tools::debugging"] +keywords = ["metrics", "facade", "macros"] + [dependencies] -metrics = { version = "^0.18", path = "../metrics", default-features = false } -metrics-util = { version = "^0.12", path = "../metrics-util", default-features = false, features = ["summary"] } +metrics = { version = "^0.21", path = "../metrics", default-features = false } +metrics-util = { version = "^0.15", path = "../metrics-util", default-features = false, features = ["summary"] } bytes = { version = "1", default-features = false } crossbeam-channel = { version = "0.5", default-features = false, features = ["std"] } -prost = { version = "0.9", default-features = false } -prost-types = { version = "0.9", default-features = false } -tui = { version = "0.16", default-features = false, features = ["termion"] } -termion = { version = "1.5", default-features = false } +prost = { version = "0.11", default-features = false } +prost-types = { version = "0.11", default-features = false } +tui = { version = "0.19", default-features = false, features = ["termion"] } +termion = { version = "2", default-features = false } chrono = { version = "0.4", default-features = false, features = ["clock"] } [build-dependencies] -prost-build = "0.9" +prost-build = "0.11" diff --git a/metrics-observer/build.rs b/metrics-observer/build.rs index 15559df8..7fe5a9ef 100644 --- a/metrics-observer/build.rs +++ b/metrics-observer/build.rs @@ -1,6 +1,6 @@ fn main() { println!("cargo:rerun-if-changed=proto/event.proto"); let mut prost_build = prost_build::Config::new(); - prost_build.btree_map(&["."]); + prost_build.btree_map(["."]); prost_build.compile_protos(&["proto/event.proto"], &["proto/"]).unwrap(); } diff --git a/metrics-observer/proto/event.proto b/metrics-observer/proto/event.proto index f24116a7..ccbe76ea 100644 --- a/metrics-observer/proto/event.proto +++ b/metrics-observer/proto/event.proto @@ -24,29 +24,16 @@ message Metric { string name = 1; google.protobuf.Timestamp timestamp = 2; map labels = 3; - oneof value { - Counter counter = 4; - Gauge gauge = 5; - Histogram histogram = 6; + oneof operation { + uint64 increment_counter = 4; + uint64 set_counter = 5; + double increment_gauge = 6; + double decrement_gauge = 7; + double set_gauge = 8; + double record_histogram = 9; } } -message Counter { - uint64 value = 1; -} - -message Gauge { - oneof value { - double absolute = 1; - double increment = 2; - double decrement = 3; - } -} - -message Histogram { - double value = 1; -} - message Event { oneof event { Metadata metadata = 1; diff --git a/metrics-observer/src/main.rs b/metrics-observer/src/main.rs index 8d6b05e4..dff15517 100644 --- a/metrics-observer/src/main.rs +++ b/metrics-observer/src/main.rs @@ -5,7 +5,7 @@ use std::{error::Error, io}; use chrono::Local; use metrics::Unit; -use termion::{event::Key, input::MouseTerminal, raw::IntoRawMode, screen::AlternateScreen}; +use termion::{event::Key, input::MouseTerminal, raw::IntoRawMode, screen::IntoAlternateScreen}; use tui::{ backend::TermionBackend, layout::{Constraint, Direction, Layout}, @@ -28,8 +28,7 @@ use self::selector::Selector; fn main() -> Result<(), Box> { let stdout = io::stdout().into_raw_mode()?; - let stdout = MouseTerminal::from(stdout); - let stdout = AlternateScreen::from(stdout); + let stdout = MouseTerminal::from(stdout).into_alternate_screen()?; let backend = TermionBackend::new(stdout); let mut terminal = Terminal::new(backend)?; diff --git a/metrics-observer/src/metrics.rs b/metrics-observer/src/metrics.rs index a5ff87e8..af5c8e72 100644 --- a/metrics-observer/src/metrics.rs +++ b/metrics-observer/src/metrics.rs @@ -16,11 +16,10 @@ mod proto { include!(concat!(env!("OUT_DIR"), "/event.proto.rs")); } -use self::proto::{ - event::Event, - metadata::{Description as DescriptionMetadata, MetricType, Unit as UnitMetadata}, - Event as EventWrapper, -}; +use proto::{event::Event, metadata::MetricType, metric::Operation, Event as ProstMessage}; + +type MetadataKey = (MetricKind, String); +type MetadataValue = (Option, Option); #[derive(Clone)] pub enum ClientState { @@ -38,7 +37,7 @@ pub enum MetricData { pub struct Client { state: Arc>, metrics: Arc>>, - metadata: Arc, Option)>>>, + metadata: Arc>>, } impl Client { @@ -93,7 +92,7 @@ struct Runner { addr: String, client_state: Arc>, metrics: Arc>>, - metadata: Arc, Option)>>>, + metadata: Arc>>, } impl Runner { @@ -101,7 +100,7 @@ impl Runner { addr: String, state: Arc>, metrics: Arc>>, - metadata: Arc, Option)>>>, + metadata: Arc>>, ) -> Runner { Runner { state: RunnerState::Disconnected, addr, client_state: state, metrics, metadata } } @@ -115,30 +114,22 @@ impl Runner { let mut state = self.client_state.lock().unwrap(); *state = ClientState::Disconnected(None); } - - // Try to connect to our target and transition into Connected. - let addr = match self.addr.to_socket_addrs() { - Ok(mut addrs) => match addrs.next() { - Some(addr) => addr, - None => { - let mut state = self.client_state.lock().unwrap(); - *state = ClientState::Disconnected(Some( - "failed to resolve specified host".to_string(), - )); - break; - } - }, - Err(_) => { - let mut state = self.client_state.lock().unwrap(); - *state = ClientState::Disconnected(Some( - "failed to resolve specified host".to_string(), - )); - break; - } + // Resolve the target address. + let Ok(mut addrs) = self.addr.to_socket_addrs() else { + let mut state = self.client_state.lock().unwrap(); + *state = ClientState::Disconnected(Some( + "failed to resolve specified host".to_string(), + )); + break; }; - match TcpStream::connect_timeout(&addr, Duration::from_secs(3)) { - Ok(stream) => RunnerState::Connected(stream), - Err(_) => RunnerState::ErrorBackoff( + // Some of the resolved addresses may be unreachable (e.g. IPv6). + // Pick the first one that works. + let maybe_stream = addrs.find_map(|addr| { + TcpStream::connect_timeout(&addr, Duration::from_secs(3)).ok() + }); + match maybe_stream { + Some(stream) => RunnerState::Connected(stream), + None => RunnerState::ErrorBackoff( "error while connecting", Duration::from_secs(3), ), @@ -172,98 +163,116 @@ impl Runner { Err(e) => eprintln!("read error: {:?}", e), }; - let event = match EventWrapper::decode_length_delimited(&mut buf) { + let message = match ProstMessage::decode_length_delimited(&mut buf) { Err(e) => { eprintln!("decode error: {:?}", e); continue; } - Ok(event) => event, + Ok(v) => v, }; - if let Some(event) = event.event { - match event { - Event::Metadata(metadata) => { - let metric_type = MetricType::from_i32(metadata.metric_type) - .expect("unknown metric type over wire"); - let metric_kind = match metric_type { - MetricType::Counter => MetricKind::Counter, - MetricType::Gauge => MetricKind::Gauge, - MetricType::Histogram => MetricKind::Histogram, - }; - let key = (metric_kind, metadata.name); - let mut mmap = self - .metadata - .write() - .expect("failed to get metadata write lock"); - let entry = mmap.entry(key).or_insert((None, None)); - let (uentry, dentry) = entry; - *uentry = metadata - .unit - .map(|u| match u { - UnitMetadata::UnitValue(us) => us, - }) - .and_then(|s| Unit::from_string(s.as_str())); - *dentry = metadata.description.map(|d| match d { - DescriptionMetadata::DescriptionValue(ds) => ds, - }); - } - Event::Metric(metric) => { - let mut labels_raw = - metric.labels.into_iter().collect::>(); - labels_raw.sort_by(|a, b| a.0.cmp(&b.0)); - let labels = labels_raw - .into_iter() - .map(|(k, v)| Label::new(k, v)) - .collect::>(); - let key_data: Key = (metric.name, labels).into(); + let event = match message.event { + Some(e) => e, + None => continue, + }; + + match event { + Event::Metadata(metadata) => { + let metric_type = MetricType::from_i32(metadata.metric_type) + .expect("unknown metric type over wire"); + let metric_kind = match metric_type { + MetricType::Counter => MetricKind::Counter, + MetricType::Gauge => MetricKind::Gauge, + MetricType::Histogram => MetricKind::Histogram, + }; + let key = (metric_kind, metadata.name); + let mut mmap = self + .metadata + .write() + .expect("failed to get metadata write lock"); + let entry = mmap.entry(key).or_insert((None, None)); + let (uentry, dentry) = entry; + *uentry = metadata + .unit + .map(|u| match u { + proto::metadata::Unit::UnitValue(u) => u, + }) + .and_then(|s| Unit::from_string(s.as_str())); + *dentry = metadata.description.map(|d| match d { + proto::metadata::Description::DescriptionValue(ds) => ds, + }); + } + Event::Metric(metric) => { + let mut labels_raw = metric.labels.into_iter().collect::>(); + labels_raw.sort_by(|a, b| a.0.cmp(&b.0)); + let labels = labels_raw + .into_iter() + .map(|(k, v)| Label::new(k, v)) + .collect::>(); + let key_data: Key = (metric.name, labels).into(); - match metric.value.expect("no metric value") { - proto::metric::Value::Counter(value) => { - let key = - CompositeKey::new(MetricKind::Counter, key_data); - let mut metrics = self.metrics.write().unwrap(); - let counter = metrics - .entry(key) - .or_insert_with(|| MetricData::Counter(0)); - if let MetricData::Counter(inner) = counter { - *inner += value.value; - } + match metric.operation.expect("no metric operation") { + Operation::IncrementCounter(value) => { + let key = CompositeKey::new(MetricKind::Counter, key_data); + let mut metrics = self.metrics.write().unwrap(); + let counter = metrics + .entry(key) + .or_insert_with(|| MetricData::Counter(0)); + if let MetricData::Counter(inner) = counter { + *inner += value; } - proto::metric::Value::Gauge(value) => { - let key = - CompositeKey::new(MetricKind::Gauge, key_data); - let mut metrics = self.metrics.write().unwrap(); - let gauge = metrics - .entry(key) - .or_insert_with(|| MetricData::Gauge(0.0)); - if let MetricData::Gauge(inner) = gauge { - match value.value { - Some(proto::gauge::Value::Absolute(val)) => { - *inner = val - } - Some(proto::gauge::Value::Increment(val)) => { - *inner += val - } - Some(proto::gauge::Value::Decrement(val)) => { - *inner -= val - } - None => {} - } - } + } + Operation::SetCounter(value) => { + let key = CompositeKey::new(MetricKind::Counter, key_data); + let mut metrics = self.metrics.write().unwrap(); + let counter = metrics + .entry(key) + .or_insert_with(|| MetricData::Counter(0)); + if let MetricData::Counter(inner) = counter { + *inner = value; + } + } + Operation::IncrementGauge(value) => { + let key = CompositeKey::new(MetricKind::Gauge, key_data); + let mut metrics = self.metrics.write().unwrap(); + let gauge = metrics + .entry(key) + .or_insert_with(|| MetricData::Gauge(0.0)); + if let MetricData::Gauge(inner) = gauge { + *inner += value; + } + } + Operation::DecrementGauge(value) => { + let key = CompositeKey::new(MetricKind::Gauge, key_data); + let mut metrics = self.metrics.write().unwrap(); + let gauge = metrics + .entry(key) + .or_insert_with(|| MetricData::Gauge(0.0)); + if let MetricData::Gauge(inner) = gauge { + *inner -= value; + } + } + Operation::SetGauge(value) => { + let key = CompositeKey::new(MetricKind::Gauge, key_data); + let mut metrics = self.metrics.write().unwrap(); + let gauge = metrics + .entry(key) + .or_insert_with(|| MetricData::Gauge(0.0)); + if let MetricData::Gauge(inner) = gauge { + *inner = value; } - proto::metric::Value::Histogram(value) => { - let key = - CompositeKey::new(MetricKind::Histogram, key_data); - let mut metrics = self.metrics.write().unwrap(); - let histogram = - metrics.entry(key).or_insert_with(|| { - let summary = Summary::with_defaults(); - MetricData::Histogram(summary) - }); + } + Operation::RecordHistogram(value) => { + let key = + CompositeKey::new(MetricKind::Histogram, key_data); + let mut metrics = self.metrics.write().unwrap(); + let histogram = metrics.entry(key).or_insert_with(|| { + let summary = Summary::with_defaults(); + MetricData::Histogram(summary) + }); - if let MetricData::Histogram(inner) = histogram { - inner.add(value.value); - } + if let MetricData::Histogram(inner) = histogram { + inner.add(value); } } } diff --git a/metrics-tracing-context/CHANGELOG.md b/metrics-tracing-context/CHANGELOG.md index 24d024d6..63ed2d4d 100644 --- a/metrics-tracing-context/CHANGELOG.md +++ b/metrics-tracing-context/CHANGELOG.md @@ -8,6 +8,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - ReleaseDate +### Added + +- Support for dynamism using `tracing::Span::record` to add label values. ([#408](https://github.com/metrics-rs/metrics/pull/408)) + +## [0.14.0] - 2023-04-16 + +### Changed + +- Bump MSRV to 1.61.0. + +## [0.13.0] - 2023-01-20 + +## [0.12.0] - 2022-07-20 + +### Changed + +- Update `metrics` to `0.20`. + +## [0.11.0] - 2022-05-30 + +### Added + +- A new label filter, `Allowlist`, to only collect labels which are present in the list. ([#288](https://github.com/metrics-rs/metrics/pull/288)) + +### Changed + +- Bumped the dependency on `metrics` to deal with a public API change. + ## [0.10.0] - 2022-01-14 ### Changed diff --git a/metrics-tracing-context/Cargo.toml b/metrics-tracing-context/Cargo.toml index 1d4cb5ff..96287598 100644 --- a/metrics-tracing-context/Cargo.toml +++ b/metrics-tracing-context/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "metrics-tracing-context" -version = "0.10.0" +version = "0.14.0" authors = ["MOZGIII "] edition = "2018" +rust-version = "1.61.0" license = "MIT" @@ -28,16 +29,18 @@ harness = false [dependencies] itoa = { version = "1", default-features = false } -metrics = { version = "^0.18", path = "../metrics" } -metrics-util = { version = "^0.12", path = "../metrics-util" } +metrics = { version = "^0.21", path = "../metrics" } +metrics-util = { version = "^0.15", path = "../metrics-util" } lockfree-object-pool = { version = "0.1.3", default-features = false } +indexmap = { version = "2.1", default-features = false, features = ["std"] } once_cell = { version = "1", default-features = false, features = ["std"] } tracing = { version = "0.1.29", default-features = false } tracing-core = { version = "0.1.21", default-features = false } tracing-subscriber = { version = "0.3.1", default-features = false, features = ["std"] } [dev-dependencies] -criterion = "0.3" -parking_lot = "0.11" +criterion = { version = "=0.3.3", default-features = false } +parking_lot = { version = "0.12.1", default-features = false } tracing = { version = "0.1.29", default-features = false, features = ["std"] } tracing-subscriber = { version = "0.3.1", default-features = false, features = ["registry"] } +itertools = { version = "0.12.0", default-features = false, features = ["use_std"] } diff --git a/metrics-tracing-context/benches/layer.rs b/metrics-tracing-context/benches/layer.rs index 7451c352..892b6668 100644 --- a/metrics-tracing-context/benches/layer.rs +++ b/metrics-tracing-context/benches/layer.rs @@ -15,9 +15,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "key"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }); group.bench_function("no integration", |b| { @@ -33,9 +35,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "key"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }) }); @@ -52,9 +56,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "key"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }) }); @@ -72,9 +78,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "key"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }) }); @@ -92,9 +100,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "key"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }) }); diff --git a/metrics-tracing-context/benches/visit.rs b/metrics-tracing-context/benches/visit.rs index a3128af9..9b6db089 100644 --- a/metrics-tracing-context/benches/visit.rs +++ b/metrics-tracing-context/benches/visit.rs @@ -1,8 +1,9 @@ use std::sync::Arc; use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use indexmap::IndexMap; use lockfree_object_pool::LinearObjectPool; -use metrics::Label; +use metrics::SharedString; use metrics_tracing_context::Labels; use once_cell::sync::OnceCell; use tracing::Metadata; @@ -13,9 +14,11 @@ use tracing_core::{ Callsite, Interest, }; -fn get_pool() -> &'static Arc>> { - static POOL: OnceCell>>> = OnceCell::new(); - POOL.get_or_init(|| Arc::new(LinearObjectPool::new(|| Vec::new(), |vec| vec.clear()))) +type Map = IndexMap; + +fn get_pool() -> &'static Arc> { + static POOL: OnceCell>> = OnceCell::new(); + POOL.get_or_init(|| Arc::new(LinearObjectPool::new(Map::new, Map::clear))) } const BATCH_SIZE: usize = 1000; diff --git a/metrics-tracing-context/src/label_filter.rs b/metrics-tracing-context/src/label_filter.rs index c8b44d2e..fa42fc80 100644 --- a/metrics-tracing-context/src/label_filter.rs +++ b/metrics-tracing-context/src/label_filter.rs @@ -2,13 +2,14 @@ use std::collections::HashSet; -use metrics::Label; +use metrics::{KeyName, Label}; /// [`LabelFilter`] trait encapsulates the ability to filter labels, i.e. /// determining whether a particular span field should be included as a label or not. pub trait LabelFilter { - /// Returns `true` if the passed label should be included in the key. - fn should_include_label(&self, label: &Label) -> bool; + /// Returns `true` if the passed `label` of the metric named `name` should + /// be included in the key. + fn should_include_label(&self, name: &KeyName, label: &Label) -> bool; } /// A [`LabelFilter`] that allows all labels. @@ -16,7 +17,7 @@ pub trait LabelFilter { pub struct IncludeAll; impl LabelFilter for IncludeAll { - fn should_include_label(&self, _label: &Label) -> bool { + fn should_include_label(&self, _name: &KeyName, _label: &Label) -> bool { true } } @@ -40,7 +41,7 @@ impl Allowlist { } impl LabelFilter for Allowlist { - fn should_include_label(&self, label: &Label) -> bool { + fn should_include_label(&self, _name: &KeyName, label: &Label) -> bool { self.label_names.contains(label.key()) } } diff --git a/metrics-tracing-context/src/lib.rs b/metrics-tracing-context/src/lib.rs index ad5acfaf..656e6449 100644 --- a/metrics-tracing-context/src/lib.rs +++ b/metrics-tracing-context/src/lib.rs @@ -45,7 +45,7 @@ //! let span = span!(Level::TRACE, "login", user); //! let _guard = span.enter(); //! -//! counter!("login_attempts", 1, "service" => "login_service"); +//! counter!("login_attempts", "service" => "login_service").increment(1); //! ``` //! //! The code above will emit a increment for a `login_attempts` counter with @@ -55,19 +55,25 @@ //! //! # Implementation //! -//! The integration layer works by capturing all fields present when a span is created and storing -//! them as an extension to the span. If a metric is emitted while a span is entered, we check that -//! span to see if it has any fields in the extension data, and if it does, we add those fields as -//! labels to the metric key. +//! The integration layer works by capturing all fields that are present when a span is created, +//! as well as fields recorded after the fact, and storing them as an extension to the span. If +//! a metric is emitted while a span is entered, any fields captured for that span will be added +//! to the metric as additional labels. //! -//! There are two important behaviors to be aware of: -//! - we only capture the fields present when the span is created -//! - we store all fields that a span has, including the fields of its parent span(s) +//! Be aware that we recursively capture the fields of a span, including fields from +//! parent spans, and use them when generating metric labels. This means that if a metric is being +//! emitted in span B, which is a child of span A, and span A has field X, and span B has field Y, +//! then the metric labels will include both field X and Y. This applies regardless of how many +//! nested spans are currently entered. //! -//! ## Lack of dynamism +//! ## Duplicate span fields //! -//! This means that if you use [`Span::record`][tracing::Span::record] to add fields to a span after -//! it has been created, those fields will not be captured and added to your metric key. +//! When span fields are captured, they are deduplicated such that only the most recent value is kept. +//! For merging parent span fields into the current span fields, the fields from the current span have +//! the highest priority. Additionally, when using [`Span::record`][tracing::Span::record] to add fields +//! to a span after it has been created, the same behavior applies. This means that recording a field +//! multiple times only keeps the most recently recorded value, including if a field was already present +//! from a parent span and is then recorded dynamically in the current span. //! //! ## Span fields and ancestry //! @@ -95,18 +101,19 @@ #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg), deny(rustdoc::broken_intra_doc_links))] -use metrics::{Counter, Gauge, Histogram, Key, KeyName, Label, Recorder, Unit}; +use metrics::{ + Counter, Gauge, Histogram, Key, KeyName, Label, Metadata, Recorder, SharedString, Unit, +}; use metrics_util::layers::Layer; pub mod label_filter; mod tracing_integration; pub use label_filter::LabelFilter; -use tracing_integration::WithContext; +use tracing_integration::Map; pub use tracing_integration::{Labels, MetricsLayer}; -/// [`TracingContextLayer`] provides an implementation of a [`Layer`][metrics_util::layers::Layer] -/// for [`TracingContext`]. +/// [`TracingContextLayer`] provides an implementation of a [`Layer`] for [`TracingContext`]. pub struct TracingContextLayer { label_filter: F, } @@ -168,32 +175,33 @@ where // doing the iteration would likely exceed the cost of simply constructing the new key. tracing::dispatcher::get_default(|dispatch| { let current = dispatch.current_span(); - if let Some(id) = current.id() { - // We're currently within a live tracing span, so see if we have an available - // metrics context to grab any fields/labels out of. - if let Some(ctx) = dispatch.downcast_ref::() { - let mut f = |new_labels: &[Label]| { - if !new_labels.is_empty() { - let (name, mut labels) = key.clone().into_parts(); - - let filtered_labels = new_labels - .iter() - .filter(|label| self.label_filter.should_include_label(label)) - .cloned(); - labels.extend(filtered_labels); - - Some(Key::from_parts(name, labels)) - } else { - None - } - }; - - // Pull in the span's fields/labels if they exist. - return ctx.with_labels(dispatch, id, &mut f); - } - } - - None + let id = current.id()?; + let ctx = dispatch.downcast_ref::()?; + + let mut f = |mut span_labels: Map| { + (!span_labels.is_empty()).then(|| { + let (name, labels) = key.clone().into_parts(); + + // Filter only span labels + span_labels.retain(|key: &SharedString, value: &mut SharedString| { + let label = Label::new(key.clone(), value.clone()); + self.label_filter.should_include_label(&name, &label) + }); + + // Overwrites labels from spans + span_labels.extend(labels.into_iter().map(Label::into_parts)); + + let labels = span_labels + .into_iter() + .map(|(key, value)| Label::new(key, value)) + .collect::>(); + + Key::from_parts(name, labels) + }) + }; + + // Pull in the span's fields/labels if they exist. + ctx.with_labels(dispatch, id, &mut f) }) } } @@ -203,33 +211,33 @@ where R: Recorder, F: LabelFilter, { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString) { self.inner.describe_counter(key_name, unit, description) } - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { self.inner.describe_gauge(key_name, unit, description) } - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString) { self.inner.describe_histogram(key_name, unit, description) } - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter { let new_key = self.enhance_key(key); let key = new_key.as_ref().unwrap_or(key); - self.inner.register_counter(key) + self.inner.register_counter(key, metadata) } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge { let new_key = self.enhance_key(key); let key = new_key.as_ref().unwrap_or(key); - self.inner.register_gauge(key) + self.inner.register_gauge(key, metadata) } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram { let new_key = self.enhance_key(key); let key = new_key.as_ref().unwrap_or(key); - self.inner.register_histogram(key) + self.inner.register_histogram(key, metadata) } } diff --git a/metrics-tracing-context/src/tracing_integration.rs b/metrics-tracing-context/src/tracing_integration.rs index e5691d24..d6e235f4 100644 --- a/metrics-tracing-context/src/tracing_integration.rs +++ b/metrics-tracing-context/src/tracing_integration.rs @@ -1,28 +1,49 @@ //! The code that integrates with the `tracing` crate. +use indexmap::IndexMap; use lockfree_object_pool::{LinearObjectPool, LinearOwnedReusable}; -use metrics::{Key, Label}; +use metrics::{Key, SharedString}; use once_cell::sync::OnceCell; +use std::cmp; use std::sync::Arc; -use std::{any::TypeId, marker::PhantomData}; use tracing_core::span::{Attributes, Id, Record}; use tracing_core::{field::Visit, Dispatch, Field, Subscriber}; use tracing_subscriber::{layer::Context, registry::LookupSpan, Layer}; -fn get_pool() -> &'static Arc>> { - static POOL: OnceCell>>> = OnceCell::new(); - POOL.get_or_init(|| Arc::new(LinearObjectPool::new(Vec::new, Vec::clear))) +pub(crate) type Map = IndexMap; + +fn get_pool() -> &'static Arc> { + static POOL: OnceCell>> = OnceCell::new(); + POOL.get_or_init(|| Arc::new(LinearObjectPool::new(Map::new, Map::clear))) } + /// Span fields mapped as metrics labels. /// /// Hidden from documentation as there is no need for end users to ever touch this type, but it must /// be public in order to be pulled in by external benchmark code. #[doc(hidden)] -pub struct Labels(pub LinearOwnedReusable>); +pub struct Labels(pub LinearOwnedReusable); impl Labels { - pub(crate) fn extend_from_labels(&mut self, other: &Labels) { - self.0.extend_from_slice(other.as_ref()); + fn extend(&mut self, other: &Labels, f: impl Fn(&mut Map, &SharedString, &SharedString)) { + let new_len = cmp::max(self.as_ref().len(), other.as_ref().len()); + let additional = new_len - self.as_ref().len(); + self.0.reserve(additional); + for (k, v) in other.as_ref() { + f(&mut self.0, k, v); + } + } + + fn extend_from_labels(&mut self, other: &Labels) { + self.extend(other, |map, k, v| { + map.entry(k.clone()).or_insert_with(|| v.clone()); + }); + } + + fn extend_from_labels_overwrite(&mut self, other: &Labels) { + self.extend(other, |map, k, v| { + map.insert(k.clone(), v.clone()); + }); } } @@ -34,108 +55,86 @@ impl Default for Labels { impl Visit for Labels { fn record_str(&mut self, field: &Field, value: &str) { - let label = Label::new(field.name(), value.to_string()); - self.0.push(label); + self.0.insert(field.name().into(), value.to_owned().into()); } fn record_bool(&mut self, field: &Field, value: bool) { - let label = Label::from_static_parts(field.name(), if value { "true" } else { "false" }); - self.0.push(label); + self.0.insert(field.name().into(), if value { "true" } else { "false" }.into()); } fn record_i64(&mut self, field: &Field, value: i64) { let mut buf = itoa::Buffer::new(); let s = buf.format(value); - let label = Label::new(field.name(), s.to_string()); - self.0.push(label); + self.0.insert(field.name().into(), s.to_owned().into()); } fn record_u64(&mut self, field: &Field, value: u64) { let mut buf = itoa::Buffer::new(); let s = buf.format(value); - let label = Label::new(field.name(), s.to_string()); - self.0.push(label); + self.0.insert(field.name().into(), s.to_owned().into()); } fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - let value_string = format!("{:?}", value); - let label = Label::new(field.name(), value_string); - self.0.push(label); + self.0.insert(field.name().into(), format!("{value:?}").into()); } } impl Labels { - fn from_attributes(attrs: &Attributes<'_>) -> Labels { + fn from_record(record: &Record) -> Labels { let mut labels = Labels::default(); - let record = Record::new(attrs.values()); record.record(&mut labels); labels } } -impl AsRef<[Label]> for Labels { - fn as_ref(&self) -> &[Label] { +impl AsRef for Labels { + fn as_ref(&self) -> &Map { &self.0 } } -pub struct WithContext { - with_labels: fn(&Dispatch, &Id, f: &mut dyn FnMut(&Labels) -> Option) -> Option, -} - -impl WithContext { - pub fn with_labels<'a>( - &self, - dispatch: &'a Dispatch, - id: &Id, - f: &mut dyn FnMut(&[Label]) -> Option, - ) -> Option { - let mut ff = |labels: &Labels| f(labels.as_ref()); - (self.with_labels)(dispatch, id, &mut ff) - } -} - /// [`MetricsLayer`] is a [`tracing_subscriber::Layer`] that captures the span /// fields and allows them to be later on used as metrics labels. -pub struct MetricsLayer { - ctx: WithContext, - _subscriber: PhantomData, +#[derive(Default)] +pub struct MetricsLayer { + with_labels: + Option Option) -> Option>, } -impl MetricsLayer -where - S: Subscriber + for<'span> LookupSpan<'span>, -{ - /// Create a new `MetricsLayer`. +impl MetricsLayer { + /// Create a new [`MetricsLayer`]. pub fn new() -> Self { - let ctx = WithContext { with_labels: Self::with_labels }; - - Self { ctx, _subscriber: PhantomData } + Self::default() } - fn with_labels( + pub(crate) fn with_labels( + &self, dispatch: &Dispatch, id: &Id, - f: &mut dyn FnMut(&Labels) -> Option, + f: &mut dyn FnMut(Map) -> Option, ) -> Option { - let subscriber = dispatch - .downcast_ref::() - .expect("subscriber should downcast to expected type; this is a bug!"); - let span = subscriber.span(id).expect("registry should have a span for the current ID"); - - let result = - if let Some(labels) = span.extensions().get::() { f(labels) } else { None }; - result + let mut ff = |labels: &Labels| f(labels.0.clone()); + (self.with_labels?)(dispatch, id, &mut ff) } } -impl Layer for MetricsLayer +impl Layer for MetricsLayer where S: Subscriber + for<'a> LookupSpan<'a>, { + fn on_layer(&mut self, _: &mut S) { + self.with_labels = Some(|dispatch, id, f| { + let subscriber = dispatch.downcast_ref::()?; + let span = subscriber.span(id)?; + + let ext = span.extensions(); + f(ext.get::()?) + }); + } + fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, cx: Context<'_, S>) { let span = cx.span(id).expect("span must already exist!"); - let mut labels = Labels::from_attributes(attrs); + let mut labels = Labels::from_record(&Record::new(attrs.values())); if let Some(parent) = span.parent() { if let Some(parent_labels) = parent.extensions().get::() { @@ -146,20 +145,15 @@ where span.extensions_mut().insert(labels); } - unsafe fn downcast_raw(&self, id: TypeId) -> Option<*const ()> { - match id { - id if id == TypeId::of::() => Some(self as *const _ as *const ()), - id if id == TypeId::of::() => Some(&self.ctx as *const _ as *const ()), - _ => None, - } - } -} + fn on_record(&self, id: &Id, values: &Record<'_>, cx: Context<'_, S>) { + let span = cx.span(id).expect("span must already exist!"); + let labels = Labels::from_record(values); -impl Default for MetricsLayer -where - S: Subscriber + for<'span> LookupSpan<'span>, -{ - fn default() -> Self { - MetricsLayer::new() + let ext = &mut span.extensions_mut(); + if let Some(existing) = ext.get_mut::() { + existing.extend_from_labels_overwrite(&labels); + } else { + ext.insert(labels); + } } } diff --git a/metrics-tracing-context/tests/integration.rs b/metrics-tracing-context/tests/integration.rs index c24b1947..526b0c63 100644 --- a/metrics-tracing-context/tests/integration.rs +++ b/metrics-tracing-context/tests/integration.rs @@ -1,4 +1,5 @@ -use metrics::{counter, Key, Label}; +use itertools::Itertools; +use metrics::{counter, Key, KeyName, Label}; use metrics_tracing_context::{LabelFilter, MetricsLayer, TracingContextLayer}; use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter}; use metrics_util::{layers::Layer, CompositeKey, MetricKind}; @@ -8,54 +9,65 @@ use tracing::{span, Level}; use tracing_subscriber::{layer::SubscriberExt, Registry}; static TEST_MUTEX: Mutex<()> = const_mutex(()); -static LOGIN_ATTEMPTS: &'static str = "login_attempts"; -static LOGIN_ATTEMPTS_NONE: &'static str = "login_attempts_no_labels"; -static LOGIN_ATTEMPTS_STATIC: &'static str = "login_attempts_static_labels"; -static LOGIN_ATTEMPTS_DYNAMIC: &'static str = "login_attempts_dynamic_labels"; -static LOGIN_ATTEMPTS_BOTH: &'static str = "login_attempts_static_and_dynamic_labels"; -static MY_COUNTER: &'static str = "my_counter"; -static USER_EMAIL: &'static [Label] = &[ +static LOGIN_ATTEMPTS: &str = "login_attempts"; +static LOGIN_ATTEMPTS_NONE: &str = "login_attempts_no_labels"; +static LOGIN_ATTEMPTS_STATIC: &str = "login_attempts_static_labels"; +static LOGIN_ATTEMPTS_DYNAMIC: &str = "login_attempts_dynamic_labels"; +static LOGIN_ATTEMPTS_BOTH: &str = "login_attempts_static_and_dynamic_labels"; +static MY_COUNTER: &str = "my_counter"; +static USER_EMAIL: &[Label] = &[ Label::from_static_parts("user", "ferris"), Label::from_static_parts("user.email", "ferris@rust-lang.org"), ]; -static EMAIL_USER: &'static [Label] = &[ +static USER_EMAIL_ATTEMPT: &[Label] = &[ + Label::from_static_parts("user", "ferris"), Label::from_static_parts("user.email", "ferris@rust-lang.org"), + Label::from_static_parts("attempt", "42"), +]; +static USER_ID: &[Label] = &[Label::from_static_parts("user.id", "42")]; +static EMAIL_USER: &[Label] = &[ Label::from_static_parts("user", "ferris"), + Label::from_static_parts("user.email", "ferris@rust-lang.org"), ]; -static SVC_ENV: &'static [Label] = &[ +static SVC_ENV: &[Label] = &[ Label::from_static_parts("service", "login_service"), Label::from_static_parts("env", "test"), ]; -static SVC_USER_EMAIL: &'static [Label] = &[ - Label::from_static_parts("service", "login_service"), +static SVC_USER_EMAIL: &[Label] = &[ Label::from_static_parts("user", "ferris"), Label::from_static_parts("user.email", "ferris@rust-lang.org"), + Label::from_static_parts("service", "login_service"), ]; -static NODE_USER_EMAIL: &'static [Label] = &[ - Label::from_static_parts("node_name", "localhost"), +static SVC_USER_EMAIL_ID: &[Label] = &[ Label::from_static_parts("user", "ferris"), Label::from_static_parts("user.email", "ferris@rust-lang.org"), -]; -static SVC_NODE_USER_EMAIL: &'static [Label] = &[ + Label::from_static_parts("user.id", "42"), Label::from_static_parts("service", "login_service"), +]; +static NODE_USER_EMAIL: &[Label] = &[ + Label::from_static_parts("user", "ferris"), + Label::from_static_parts("user.email", "ferris@rust-lang.org"), Label::from_static_parts("node_name", "localhost"), +]; +static SVC_NODE_USER_EMAIL: &[Label] = &[ Label::from_static_parts("user", "ferris"), Label::from_static_parts("user.email", "ferris@rust-lang.org"), + Label::from_static_parts("service", "login_service"), + Label::from_static_parts("node_name", "localhost"), ]; -static COMBINED_LABELS: &'static [Label] = &[ +static COMBINED_LABELS: &[Label] = &[ Label::from_static_parts("shared_field", "inner"), Label::from_static_parts("inner_specific", "foo"), Label::from_static_parts("inner_specific_dynamic", "foo_dynamic"), - Label::from_static_parts("shared_field", "outer"), Label::from_static_parts("outer_specific", "bar"), Label::from_static_parts("outer_specific_dynamic", "bar_dynamic"), ]; -static SAME_CALLSITE_PATH_1: &'static [Label] = &[ +static SAME_CALLSITE_PATH_1: &[Label] = &[ Label::from_static_parts("shared_field", "path1"), Label::from_static_parts("path1_specific", "foo"), Label::from_static_parts("path1_specific_dynamic", "foo_dynamic"), ]; -static SAME_CALLSITE_PATH_2: &'static [Label] = &[ +static SAME_CALLSITE_PATH_2: &[Label] = &[ Label::from_static_parts("shared_field", "path2"), Label::from_static_parts("path2_specific", "bar"), Label::from_static_parts("path2_specific_dynamic", "bar_dynamic"), @@ -78,7 +90,7 @@ where let snapshotter = recorder.snapshotter(); let recorder = layer.layer(recorder); - metrics::clear_recorder(); + unsafe { metrics::clear_recorder() }; metrics::set_boxed_recorder(Box::new(recorder)).expect("failed to install recorder"); let test_guard = @@ -95,7 +107,7 @@ fn test_basic_functionality() { let span = span!(Level::TRACE, "login", user, user.email = email); let _guard = span.enter(); - counter!("login_attempts", 1, "service" => "login_service"); + counter!("login_attempts", "service" => "login_service").increment(1); let snapshot = snapshotter.snapshot().into_vec(); @@ -110,7 +122,200 @@ fn test_basic_functionality() { None, DebugValue::Counter(1), )] - ) + ); +} + +#[test] +fn test_basic_functionality_record() { + let (_guard, snapshotter) = setup(TracingContextLayer::all()); + + let user = "ferris"; + let email = "ferris@rust-lang.org"; + let span = span!( + Level::TRACE, + "login", + user, + user.email = email, + user.id = tracing_core::field::Empty, + ); + let _guard = span.enter(); + + span.record("user.id", 42); + counter!("login_attempts", "service" => "login_service").increment(1); + + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!( + snapshot, + vec![( + CompositeKey::new( + MetricKind::Counter, + Key::from_static_parts(LOGIN_ATTEMPTS, SVC_USER_EMAIL_ID) + ), + None, + None, + DebugValue::Counter(1), + )] + ); +} + +#[test] +fn test_basic_functionality_then_record() { + let (_guard, snapshotter) = setup(TracingContextLayer::all()); + + let user = "ferris"; + let email = "ferris@rust-lang.org"; + let span = span!( + Level::TRACE, + "login", + user, + user.email = email, + user.id = tracing_core::field::Empty, + ); + let _guard = span.enter(); + let mut snapshots = vec![]; + { + counter!("login_attempts", "service" => "login_service").increment(1); + + let snapshot = snapshotter.snapshot().into_vec(); + + snapshots.push(( + CompositeKey::new( + MetricKind::Counter, + Key::from_static_parts(LOGIN_ATTEMPTS, SVC_USER_EMAIL), + ), + None, + None, + DebugValue::Counter(1), + )); + + assert_eq!(snapshot, snapshots); + } + span.record("user.id", 42); + { + counter!("login_attempts", "service" => "login_service").increment(1); + + let snapshot = snapshotter.snapshot().into_vec(); + + snapshots.push(( + CompositeKey::new( + MetricKind::Counter, + Key::from_static_parts(LOGIN_ATTEMPTS, SVC_USER_EMAIL_ID), + ), + None, + None, + DebugValue::Counter(1), + )); + + assert_eq!(snapshot, snapshots); + } +} + +#[test] +fn test_rerecord() { + static USER_ID_42: &[Label] = &[Label::from_static_parts("user.id", "42")]; + static USER_ID_123: &[Label] = &[Label::from_static_parts("user.id", "123")]; + + let (_guard, snapshotter) = setup(TracingContextLayer::all()); + + let span = span!(Level::TRACE, "login", user.id = tracing_core::field::Empty); + let _guard = span.enter(); + + span.record("user.id", 42); + counter!("login_attempts").increment(1); + + span.record("user.id", 123); + counter!("login_attempts").increment(1); + + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!( + snapshot, + vec![ + ( + CompositeKey::new( + MetricKind::Counter, + Key::from_static_parts(LOGIN_ATTEMPTS, USER_ID_42) + ), + None, + None, + DebugValue::Counter(1), + ), + ( + CompositeKey::new( + MetricKind::Counter, + Key::from_static_parts(LOGIN_ATTEMPTS, USER_ID_123) + ), + None, + None, + DebugValue::Counter(1), + ) + ] + ); +} + +#[test] +fn test_loop() { + let (_guard, snapshotter) = setup(TracingContextLayer::all()); + + let user = "ferris"; + let email = "ferris@rust-lang.org"; + let span = span!( + Level::TRACE, + "login", + user, + user.email = email, + attempt = tracing_core::field::Empty, + ); + let _guard = span.enter(); + + for attempt in 1..=42 { + span.record("attempt", attempt); + } + counter!("login_attempts").increment(1); + + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!( + snapshot, + vec![( + CompositeKey::new( + MetricKind::Counter, + Key::from_static_parts(LOGIN_ATTEMPTS, USER_EMAIL_ATTEMPT) + ), + None, + None, + DebugValue::Counter(1), + )] + ); +} + +#[test] +fn test_record_does_not_overwrite() { + static USER_ID_42: &[Label] = &[Label::from_static_parts("user.id", "42")]; + + let (_guard, snapshotter) = setup(TracingContextLayer::all()); + + let span = span!(Level::TRACE, "login", user.id = tracing_core::field::Empty); + let _guard = span.enter(); + + span.record("user.id", 666); + counter!("login_attempts", "user.id" => "42").increment(1); + + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!( + snapshot, + vec![( + CompositeKey::new( + MetricKind::Counter, + Key::from_static_parts(LOGIN_ATTEMPTS, USER_ID_42) + ), + None, + None, + DebugValue::Counter(1), + )] + ); } #[test] @@ -123,15 +328,19 @@ fn test_macro_forms() { let _guard = span.enter(); // No labels. - counter!("login_attempts_no_labels", 1); + counter!("login_attempts_no_labels").increment(1); // Static labels only. - counter!("login_attempts_static_labels", 1, "service" => "login_service"); + counter!("login_attempts_static_labels", "service" => "login_service").increment(1); // Dynamic labels only. let node_name = "localhost".to_string(); - counter!("login_attempts_dynamic_labels", 1, "node_name" => node_name.clone()); + counter!("login_attempts_dynamic_labels", "node_name" => node_name.clone()).increment(1); // Static and dynamic. - counter!("login_attempts_static_and_dynamic_labels", 1, - "service" => "login_service", "node_name" => node_name.clone()); + counter!( + "login_attempts_static_and_dynamic_labels", + "service" => "login_service", + "node_name" => node_name, + ) + .increment(1); let snapshot = snapshotter.snapshot().into_vec(); @@ -175,7 +384,7 @@ fn test_macro_forms() { DebugValue::Counter(1), ), ] - ) + ); } #[test] @@ -185,7 +394,7 @@ fn test_no_labels() { let span = span!(Level::TRACE, "login"); let _guard = span.enter(); - counter!("login_attempts", 1); + counter!("login_attempts").increment(1); let snapshot = snapshotter.snapshot().into_vec(); @@ -197,7 +406,30 @@ fn test_no_labels() { None, DebugValue::Counter(1), )] - ) + ); +} + +#[test] +fn test_no_labels_record() { + let (_guard, snapshotter) = setup(TracingContextLayer::all()); + + let span = span!(Level::TRACE, "login", user.id = tracing_core::field::Empty); + let _guard = span.enter(); + + span.record("user.id", 42); + counter!("login_attempts").increment(1); + + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!( + snapshot, + vec![( + CompositeKey::new(MetricKind::Counter, Key::from_static_parts(LOGIN_ATTEMPTS, USER_ID)), + None, + None, + DebugValue::Counter(1), + )] + ); } #[test] @@ -205,7 +437,7 @@ fn test_multiple_paths_to_the_same_callsite() { let (_guard, snapshotter) = setup(TracingContextLayer::all()); let shared_fn = || { - counter!("my_counter", 1); + counter!("my_counter").increment(1); }; let path1 = || { @@ -261,7 +493,7 @@ fn test_multiple_paths_to_the_same_callsite() { DebugValue::Counter(1), ) ] - ) + ); } #[test] @@ -279,7 +511,7 @@ fn test_nested_spans() { ); let _guard = span.enter(); - counter!("my_counter", 1); + counter!("my_counter").increment(1); }; let outer = || { @@ -317,7 +549,7 @@ fn test_nested_spans() { struct OnlyUser; impl LabelFilter for OnlyUser { - fn should_include_label(&self, label: &Label) -> bool { + fn should_include_label(&self, _name: &KeyName, label: &Label) -> bool { label.key() == "user" } } @@ -331,7 +563,7 @@ fn test_label_filtering() { let span = span!(Level::TRACE, "login", user, user.email_span = email); let _guard = span.enter(); - counter!("login_attempts", 1, "user.email" => "ferris@rust-lang.org"); + counter!("login_attempts", "user.email" => "ferris@rust-lang.org").increment(1); let snapshot = snapshotter.snapshot().into_vec(); @@ -346,12 +578,12 @@ fn test_label_filtering() { None, DebugValue::Counter(1), )] - ) + ); } #[test] fn test_label_allowlist() { - let (_guard, snapshotter) = setup(TracingContextLayer::only_allow(&["env", "service"])); + let (_guard, snapshotter) = setup(TracingContextLayer::only_allow(["env", "service"])); let user = "ferris"; let email = "ferris@rust-lang.org"; @@ -365,7 +597,7 @@ fn test_label_allowlist() { ); let _guard = span.enter(); - counter!("login_attempts", 1); + counter!("login_attempts").increment(1); let snapshot = snapshotter.snapshot().into_vec(); @@ -377,5 +609,187 @@ fn test_label_allowlist() { None, DebugValue::Counter(1), )] - ) + ); +} + +#[test] +fn test_all_permutations() { + let perms = (0..9).map(|_| [false, true]).multi_cartesian_product(); + + for v in perms { + let [metric_has_labels, in_span, span_has_fields, span_field_same_as_metric, span_has_parent, parent_field_same_as_span, span_field_is_empty, record_field, increment_before_recording] = + v[..] + else { + unreachable!("{:?}, {}", v, v.len()); + }; + + test( + metric_has_labels, + in_span, + span_has_fields, + span_field_same_as_metric, + span_has_parent, + parent_field_same_as_span, + span_field_is_empty, + record_field, + increment_before_recording, + ); + } +} + +#[allow(clippy::fn_params_excessive_bools, clippy::too_many_arguments, clippy::too_many_lines)] +fn test( + metric_has_labels: bool, + in_span: bool, + span_has_fields: bool, + span_field_same_as_metric: bool, + span_has_parent: bool, + parent_field_same_as_span: bool, + span_field_is_empty: bool, + record_field: bool, + increment_before_recording: bool, +) { + let (_guard, snapshotter) = setup(TracingContextLayer::all()); + + let parent = if span_field_same_as_metric && parent_field_same_as_span { + tracing::trace_span!("outer", user.email = "changed@domain.com") + } else { + tracing::trace_span!("outer", user.id = 999) + }; + + let _guard = span_has_parent.then(|| parent.enter()); + + let span = if span_has_fields { + match (span_field_same_as_metric, span_field_is_empty) { + (false, false) => tracing::trace_span!("login", user.id = 666), + (false, true) => tracing::trace_span!("login", user.id = tracing_core::field::Empty), + (true, false) => tracing::trace_span!("login", user.email = "user@domain.com"), + (true, true) => tracing::trace_span!("login", user.email = tracing_core::field::Empty), + } + } else { + tracing::trace_span!("login") + }; + + let _guard = in_span.then(|| span.enter()); + + let increment = || { + if metric_has_labels { + counter!("login_attempts", "user.email" => "ferris@rust-lang.org").increment(1); + } else { + counter!("login_attempts").increment(1); + } + }; + + if increment_before_recording { + increment(); + } + + if record_field { + span.record("user.id", 42); + } + + increment(); + + let snapshot = snapshotter.snapshot().into_vec(); + + let mut expected = vec![]; + + if in_span + && span_has_fields + && !span_field_same_as_metric + && record_field + && increment_before_recording + { + expected.push(( + CompositeKey::new( + MetricKind::Counter, + Key::from_parts( + LOGIN_ATTEMPTS, + IntoIterator::into_iter([ + (span_has_parent || !span_field_is_empty).then(|| { + Label::new("user.id", if span_field_is_empty { "999" } else { "666" }) + }), + metric_has_labels.then(|| Label::new("user.email", "ferris@rust-lang.org")), + ]) + .flatten() + .collect::>(), + ), + ), + None, + None, + DebugValue::Counter(1), + )); + } + + let in_span_with_metric_field = + in_span && span_has_fields && span_field_same_as_metric && !span_field_is_empty; + let has_other_labels = !(!span_has_parent + && (!in_span + || (span_field_same_as_metric || !span_has_fields) + || (!record_field && span_field_is_empty))) + && !(span_field_same_as_metric && parent_field_same_as_span) + && !in_span_with_metric_field; + + expected.push(( + CompositeKey::new( + MetricKind::Counter, + Key::from_parts( + LOGIN_ATTEMPTS, + IntoIterator::into_iter([ + (metric_has_labels && !has_other_labels) + .then(|| Label::new("user.email", "ferris@rust-lang.org")), + (!metric_has_labels + && (in_span_with_metric_field + || span_field_same_as_metric + && span_has_parent + && parent_field_same_as_span)) + .then(|| { + if in_span_with_metric_field { + Label::new("user.email", "user@domain.com") + } else { + Label::new("user.email", "changed@domain.com") + } + }), + if in_span && span_has_fields && !span_field_same_as_metric && record_field { + Some(Label::new("user.id", "42")) + } else if in_span + && span_has_fields + && !span_field_same_as_metric + && !span_field_is_empty + && !record_field + { + Some(Label::new("user.id", "666")) + } else if (!in_span || !span_has_fields || span_field_same_as_metric) + && (!span_field_same_as_metric || !parent_field_same_as_span) + && span_has_parent + || span_has_parent + && span_field_is_empty + && !record_field + && !span_field_same_as_metric + { + Some(Label::new("user.id", "999")) + } else { + None + }, + (metric_has_labels && has_other_labels) + .then(|| Label::new("user.email", "ferris@rust-lang.org")), + ]) + .flatten() + .collect::>(), + ), + ), + None, + None, + DebugValue::Counter( + if !increment_before_recording + || in_span && span_has_fields && !span_field_same_as_metric && record_field + { + 1 + } else { + 2 + }, + ), + )); + + assert_eq!(snapshot, expected); } diff --git a/metrics-util/CHANGELOG.md b/metrics-util/CHANGELOG.md index 9fae5251..a666c911 100644 --- a/metrics-util/CHANGELOG.md +++ b/metrics-util/CHANGELOG.md @@ -9,6 +9,55 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - ReleaseDate +### Fixed + +- Fixed the `Debug` implementation for `bucket::Block` which represented both an unsafe and + logically incorrect usage of `crossbeam-epoch.` + +## [0.15.1] - 2023-07-02 + +### Added + +- Added a new helper type, `RecoverableRecorder`, that allows installing a recorder and then + recovering it later. + +### Changed + +- Update `aho-corasick` to `1.0`. +- Pinned `hashbrown` to `0.13.1` to avoid MSRV bump. + +## [0.15.0] - 2023-04-16 + +### Changed + +- Bump MSRV to 1.61.0. +- Switch to `metrics`-exposed version of `AtomicU64`. + +## [0.14.0] - 2022-07-20 + +### Changed + +- Updated `sketches-ddsketch` to `0.2.0`. +- Switched to using `portable_atomic` for 64-bit atomics on more architectures. (#313) + +## [0.13.0] - 2022-05-30 + +### Fixed + +- In `Summary`, some quantiles were previously mapped to an incorrect rank at low sample counts, leading to large swings + in estimated values. ([#304](https://github.com/metrics-rs/metrics/pull/304)) + +### Changed + +- Bumped the dependency on `metrics` to deal with a public API change. + +## [0.12.1] - 2022-05-02 + +### Added + +- A new per-thread mode for `DebuggingRecorder` that allows recording metrics on a per-thread basis to better supporting + the testing of metrics in user applications where many tests are concurrently emitting metrics. + ## [0.12.0] - 2022-03-10 ### Added diff --git a/metrics-util/Cargo.toml b/metrics-util/Cargo.toml index f60634b5..373960d5 100644 --- a/metrics-util/Cargo.toml +++ b/metrics-util/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "metrics-util" -version = "0.12.0" +version = "0.15.1" authors = ["Toby Lawrence "] edition = "2018" +rust-version = "1.61.0" license = "MIT" @@ -46,33 +47,33 @@ name = "bucket-crusher" required-features = ["handles"] [dependencies] -metrics = { version = "^0.18", path = "../metrics" } +metrics = { version = "^0.21", path = "../metrics" } crossbeam-epoch = { version = "0.9.2", default-features = false, optional = true, features = ["alloc", "std"] } crossbeam-utils = { version = "0.8", default-features = false, optional = true } -atomic-shim = { version = "0.2", default-features = false, optional = true } -aho-corasick = { version = "0.7", default-features = false, optional = true, features = ["std"] } +aho-corasick = { version = "1", default-features = false, optional = true, features = ["std"] } indexmap = { version = "1", default-features = false, optional = true } -parking_lot = { version = "0.11", default-features = false, optional = true } -quanta = { version = "0.9.3", default-features = false, optional = true } -sketches-ddsketch = { version = "0.1", default-features = false, optional = true } +quanta = { version = "0.12", default-features = false, optional = true } +sketches-ddsketch = { version = "0.2", default-features = false, optional = true } radix_trie = { version = "0.2", default-features = false, optional = true } -ordered-float = { version = "2.0", default-features = false, optional = true } +ordered-float = { version = "4.2", default-features = false, optional = true } num_cpus = { version = "1", default-features = false, optional = true } -ahash = { version = "0.7", default-features = false, optional = true } -hashbrown = { version = "0.11", default-features = false, optional = true, features = ["ahash"] } +ahash = { version = "0.8", default-features = false, optional = true } +hashbrown = { version = "=0.13.1", default-features = false, optional = true, features = ["ahash"] } [dev-dependencies] approx = "0.5" -criterion = { version = "0.3", default-features = false, features = ["html_reports", "cargo_bench_support"] } +criterion = { version = "=0.3.3", default-features = false } rand = { version = "0.8", features = ["small_rng"] } rand_distr = "0.4" getopts = "0.2" hdrhistogram = { version = "7.2", default-features = false } -sketches-ddsketch = "0.1" +sketches-ddsketch = "0.2" ndarray = "0.15" ndarray-stats = "0.5" noisy_float = "0.2" -ordered-float = "2.0" +ordered-float = "4.2" +predicates-core = "=1.0.5" +predicates-tree = "=1.0.7" tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "ansi"] } crossbeam-queue = "0.3" @@ -88,5 +89,5 @@ layers = ["layer-filter", "layer-router"] layer-filter = ["aho-corasick"] layer-router = ["radix_trie"] summary = ["sketches-ddsketch"] -recency = ["parking_lot", "registry", "quanta"] -registry = ["atomic-shim", "crossbeam-epoch", "crossbeam-utils", "handles", "hashbrown", "num_cpus", "parking_lot"] +recency = ["registry", "quanta"] +registry = ["crossbeam-epoch", "crossbeam-utils", "handles", "hashbrown", "num_cpus"] diff --git a/metrics-util/benches/filter.rs b/metrics-util/benches/filter.rs index e2b97515..402b24be 100644 --- a/metrics-util/benches/filter.rs +++ b/metrics-util/benches/filter.rs @@ -18,9 +18,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "tokio.foo"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }); group.bench_function("no match", |b| { @@ -30,9 +32,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "hyper.foo"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }); group.bench_function("noop recorder overhead (increment_counter)", |b| { @@ -40,9 +44,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "tokio.foo"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }); } diff --git a/metrics-util/benches/prefix.rs b/metrics-util/benches/prefix.rs index 1bf32cb0..ce2beb3a 100644 --- a/metrics-util/benches/prefix.rs +++ b/metrics-util/benches/prefix.rs @@ -10,9 +10,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "simple_key"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }); group.bench_function("noop recorder overhead (increment_counter)", |b| { @@ -20,9 +22,11 @@ fn layer_benchmark(c: &mut Criterion) { static KEY_NAME: &'static str = "simple_key"; static KEY_LABELS: [Label; 1] = [Label::from_static_parts("foo", "bar")]; static KEY_DATA: Key = Key::from_static_parts(&KEY_NAME, &KEY_LABELS); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&KEY_DATA); + let _ = recorder.register_counter(&KEY_DATA, &METADATA); }) }); group.finish(); diff --git a/metrics-util/benches/router.rs b/metrics-util/benches/router.rs index 41942a96..5714108c 100644 --- a/metrics-util/benches/router.rs +++ b/metrics-util/benches/router.rs @@ -9,9 +9,11 @@ fn layer_benchmark(c: &mut Criterion) { group.bench_function("default target (via mask)", |b| { let recorder = RouterBuilder::from_recorder(NoopRecorder).build(); let key = Key::from_name("test_key"); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&key); + let _ = recorder.register_counter(&key, &METADATA); }) }); group.bench_function("default target (via fallback)", |b| { @@ -19,9 +21,11 @@ fn layer_benchmark(c: &mut Criterion) { builder.add_route(MetricKindMask::COUNTER, "override", NoopRecorder); let recorder = builder.build(); let key = Key::from_name("normal_key"); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&key); + let _ = recorder.register_counter(&key, &METADATA); }) }); group.bench_function("routed target", |b| { @@ -29,9 +33,11 @@ fn layer_benchmark(c: &mut Criterion) { builder.add_route(MetricKindMask::COUNTER, "override", NoopRecorder); let recorder = builder.build(); let key = Key::from_name("override_key"); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); b.iter(|| { - let _ = recorder.register_counter(&key); + let _ = recorder.register_counter(&key, &METADATA); }) }); } diff --git a/metrics-util/src/bucket.rs b/metrics-util/src/bucket.rs index 355323ad..3174522d 100644 --- a/metrics-util/src/bucket.rs +++ b/metrics-util/src/bucket.rs @@ -1,4 +1,4 @@ -use crossbeam_epoch::{pin as epoch_pin, unprotected, Atomic, Guard, Owned, Shared}; +use crossbeam_epoch::{pin as epoch_pin, Atomic, Guard, Owned, Shared}; use crossbeam_utils::Backoff; use std::{ cell::UnsafeCell, @@ -153,7 +153,8 @@ impl Drop for Block { impl std::fmt::Debug for Block { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let has_next = unsafe { !self.next.load(Ordering::Acquire, unprotected()).is_null() }; + let guard = &epoch_pin(); + let has_next = !self.next.load(Ordering::Acquire, guard).is_null(); f.debug_struct("Block") .field("type", &std::any::type_name::()) .field("block_size", &BLOCK_SIZE) diff --git a/metrics-util/src/debugging.rs b/metrics-util/src/debugging.rs index 0f75e058..31ee34bb 100644 --- a/metrics-util/src/debugging.rs +++ b/metrics-util/src/debugging.rs @@ -4,18 +4,29 @@ //! and core parts of the `metrics` ecosystem, they can be beneficial for in-process collecting of //! metrics in some limited cases. -use core::hash::Hash; -use std::sync::atomic::Ordering; -use std::sync::{Arc, Mutex}; -use std::{collections::HashMap, fmt::Debug}; +use std::{ + cell::RefCell, + collections::HashMap, + fmt::Debug, + hash::Hash, + sync::{atomic::Ordering, Arc, Mutex}, +}; -use crate::registry::AtomicStorage; -use crate::{kind::MetricKind, registry::Registry, CompositeKey}; +use crate::{ + kind::MetricKind, + registry::{AtomicStorage, Registry}, + CompositeKey, +}; use indexmap::IndexMap; -use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; use ordered_float::OrderedFloat; +thread_local! { + /// A per-thread version of the debugging registry/state used only when the debugging recorder is configured to run in per-thread mode. + static PER_THREAD_INNER: RefCell> = RefCell::new(None); +} + /// A composite key name that stores both the metric key name and the metric kind. #[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] struct CompositeKeyName(MetricKind, KeyName); @@ -28,14 +39,14 @@ impl CompositeKeyName { } /// A point-in-time snapshot of all metrics in [`DebuggingRecorder`]. -pub struct Snapshot(Vec<(CompositeKey, Option, Option<&'static str>, DebugValue)>); +pub struct Snapshot(Vec<(CompositeKey, Option, Option, DebugValue)>); impl Snapshot { /// Converts this snapshot to a mapping of metric data, keyed by the metric key itself. #[allow(clippy::mutable_key_type)] pub fn into_hashmap( self, - ) -> HashMap, Option<&'static str>, DebugValue)> { + ) -> HashMap, Option, DebugValue)> { self.0 .into_iter() .map(|(k, unit, desc, value)| (k, (unit, desc, value))) @@ -43,7 +54,7 @@ impl Snapshot { } /// Converts this snapshot to a vector of metric data tuples. - pub fn into_vec(self) -> Vec<(CompositeKey, Option, Option<&'static str>, DebugValue)> { + pub fn into_vec(self) -> Vec<(CompositeKey, Option, Option, DebugValue)> { self.0 } } @@ -59,11 +70,25 @@ pub enum DebugValue { Histogram(Vec>), } +struct Inner { + registry: Registry, + seen: Mutex>, + metadata: Mutex, SharedString)>>, +} + +impl Inner { + fn new() -> Self { + Self { + registry: Registry::atomic(), + seen: Mutex::new(IndexMap::new()), + metadata: Mutex::new(IndexMap::new()), + } + } +} + /// Captures point-in-time snapshots of [`DebuggingRecorder`]. pub struct Snapshotter { - registry: Arc>, - seen: Arc>>, - metadata: Arc, &'static str)>>>, + inner: Arc, } impl Snapshotter { @@ -71,12 +96,12 @@ impl Snapshotter { pub fn snapshot(&self) -> Snapshot { let mut snapshot = Vec::new(); - let counters = self.registry.get_counter_handles(); - let gauges = self.registry.get_gauge_handles(); - let histograms = self.registry.get_histogram_handles(); + let counters = self.inner.registry.get_counter_handles(); + let gauges = self.inner.registry.get_gauge_handles(); + let histograms = self.inner.registry.get_histogram_handles(); - let seen = self.seen.lock().expect("seen lock poisoned").clone(); - let metadata = self.metadata.lock().expect("metadata lock poisoned").clone(); + let seen = self.inner.seen.lock().expect("seen lock poisoned").clone(); + let metadata = self.inner.metadata.lock().expect("metadata lock poisoned").clone(); for (ck, _) in seen.into_iter() { let value = match ck.kind() { @@ -97,8 +122,7 @@ impl Snapshotter { let ckn = CompositeKeyName::new(ck.kind(), ck.key().name().to_string().into()); let (unit, desc) = metadata .get(&ckn) - .copied() - .map(|(u, d)| (u, Some(d))) + .map(|(u, d)| (u.to_owned(), Some(d.to_owned()))) .unwrap_or_else(|| (None, None)); // If there's no value for the key, that means the metric was only ever described, and @@ -110,6 +134,58 @@ impl Snapshotter { Snapshot(snapshot) } + + /// Takes a snapshot of the recorder for the current thread only. + /// + /// If no registry exists for the current thread, `None` is returned. Otherwise, `Some(snapshot)` is returned. + pub fn current_thread_snapshot() -> Option { + PER_THREAD_INNER.with(|maybe_inner| match maybe_inner.borrow().as_ref() { + None => None, + Some(inner) => { + let mut snapshot = Vec::new(); + + let counters = inner.registry.get_counter_handles(); + let gauges = inner.registry.get_gauge_handles(); + let histograms = inner.registry.get_histogram_handles(); + + let seen = inner.seen.lock().expect("seen lock poisoned").clone(); + let metadata = inner.metadata.lock().expect("metadata lock poisoned").clone(); + + for (ck, _) in seen.into_iter() { + let value = match ck.kind() { + MetricKind::Counter => counters + .get(ck.key()) + .map(|c| DebugValue::Counter(c.load(Ordering::SeqCst))), + MetricKind::Gauge => gauges.get(ck.key()).map(|g| { + let value = f64::from_bits(g.load(Ordering::SeqCst)); + DebugValue::Gauge(value.into()) + }), + MetricKind::Histogram => histograms.get(ck.key()).map(|h| { + let mut values = Vec::new(); + h.clear_with(|xs| { + values.extend(xs.iter().map(|f| OrderedFloat::from(*f))) + }); + DebugValue::Histogram(values) + }), + }; + + let ckn = CompositeKeyName::new(ck.kind(), ck.key().name().to_string().into()); + let (unit, desc) = metadata + .get(&ckn) + .map(|(u, d)| (u.to_owned(), Some(d.to_owned()))) + .unwrap_or_else(|| (None, None)); + + // If there's no value for the key, that means the metric was only ever described, and + // not registered, so don't emit it. + if let Some(value) = value { + snapshot.push((ck, unit, desc, value)); + } + } + + Some(Snapshot(snapshot)) + } + }) + } } /// A simplistic recorder that can be installed and used for debugging or testing. @@ -117,42 +193,81 @@ impl Snapshotter { /// Callers can easily take snapshots of the metrics at any given time and get access /// to the raw values. pub struct DebuggingRecorder { - registry: Arc>, - seen: Arc>>, - metadata: Arc, &'static str)>>>, + inner: Arc, + is_per_thread: bool, } impl DebuggingRecorder { /// Creates a new `DebuggingRecorder`. pub fn new() -> DebuggingRecorder { - DebuggingRecorder { - registry: Arc::new(Registry::atomic()), - seen: Arc::new(Mutex::new(IndexMap::new())), - metadata: Arc::new(Mutex::new(IndexMap::new())), - } + DebuggingRecorder { inner: Arc::new(Inner::new()), is_per_thread: false } + } + + /// Creates a new `DebuggingRecorder` in per-thread mode. + /// + /// This sends all metrics to a per-thread registry, such that the snapshotter will only see metrics emitted in the + /// thread that the `Snapshotter` is used from. Additionally, as keeping a reference to the original `Snapshotter` + /// around can be tricky, [`Snapshotter::current_thread_snapshot`] can be used to get all of the metrics currently + /// present in the registry for the calling thread, if any were emitted. + /// + /// Please note that this recorder must still be installed, but it can be installed multiple times (if the error + /// from `install` is ignored) without clearing or removing any of the existing per-thread metrics, so it's safe to + /// re-create and re-install multiple times in the same test binary if necessary. + pub fn per_thread() -> DebuggingRecorder { + DebuggingRecorder { inner: Arc::new(Inner::new()), is_per_thread: true } } /// Gets a `Snapshotter` attached to this recorder. pub fn snapshotter(&self) -> Snapshotter { - Snapshotter { - registry: self.registry.clone(), - seen: self.seen.clone(), - metadata: self.metadata.clone(), - } + Snapshotter { inner: Arc::clone(&self.inner) } } - fn describe_metric(&self, rkey: CompositeKeyName, unit: Option, desc: &'static str) { - let mut metadata = self.metadata.lock().expect("metadata lock poisoned"); - let (uentry, dentry) = metadata.entry(rkey).or_insert((None, desc)); - if unit.is_some() { - *uentry = unit; + fn describe_metric(&self, rkey: CompositeKeyName, unit: Option, desc: SharedString) { + if self.is_per_thread { + PER_THREAD_INNER.with(|cell| { + // Create the inner state if it doesn't yet exist. + // + // SAFETY: It's safe to use `borrow_mut` here, even though the parent method is `&self`, as this is a + // per-thread invocation, so no other caller could possibly be holding a referenced, immutable or + // mutable, at the same time. + let mut maybe_inner = cell.borrow_mut(); + let inner = maybe_inner.get_or_insert_with(Inner::new); + + let mut metadata = inner.metadata.lock().expect("metadata lock poisoned"); + let (uentry, dentry) = metadata.entry(rkey).or_insert((None, desc.to_owned())); + if unit.is_some() { + *uentry = unit; + } + *dentry = desc.to_owned(); + }); + } else { + let mut metadata = self.inner.metadata.lock().expect("metadata lock poisoned"); + let (uentry, dentry) = metadata.entry(rkey).or_insert((None, desc.to_owned())); + if unit.is_some() { + *uentry = unit; + } + *dentry = desc; } - *dentry = desc; } fn track_metric(&self, ckey: CompositeKey) { - let mut seen = self.seen.lock().expect("seen lock poisoned"); - seen.insert(ckey, ()); + if self.is_per_thread { + PER_THREAD_INNER.with(|cell| { + // Create the inner state if it doesn't yet exist. + // + // SAFETY: It's safe to use `borrow_mut` here, even though the parent method is `&self`, as this is a + // per-thread invocation, so no other caller could possibly be holding a referenced, immutable or + // mutable, at the same time. + let mut maybe_inner = cell.borrow_mut(); + let inner = maybe_inner.get_or_insert_with(Inner::new); + + let mut seen = inner.seen.lock().expect("seen lock poisoned"); + seen.insert(ckey, ()); + }); + } else { + let mut seen = self.inner.seen.lock().expect("seen lock poisoned"); + seen.insert(ckey, ()); + } } /// Installs this recorder as the global recorder. @@ -162,37 +277,82 @@ impl DebuggingRecorder { } impl Recorder for DebuggingRecorder { - fn describe_counter(&self, key: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key: KeyName, unit: Option, description: SharedString) { let ckey = CompositeKeyName::new(MetricKind::Counter, key); self.describe_metric(ckey, unit, description); } - fn describe_gauge(&self, key: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key: KeyName, unit: Option, description: SharedString) { let ckey = CompositeKeyName::new(MetricKind::Gauge, key); self.describe_metric(ckey, unit, description); } - fn describe_histogram(&self, key: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key: KeyName, unit: Option, description: SharedString) { let ckey = CompositeKeyName::new(MetricKind::Histogram, key); self.describe_metric(ckey, unit, description); } - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { let ckey = CompositeKey::new(MetricKind::Counter, key.clone()); self.track_metric(ckey); - self.registry.get_or_create_counter(key, |c| Counter::from_arc(c.clone())) + + if self.is_per_thread { + PER_THREAD_INNER.with(move |cell| { + // Create the inner state if it doesn't yet exist. + // + // SAFETY: It's safe to use `borrow_mut` here, even though the parent method is `&self`, as this is a + // per-thread invocation, so no other caller could possibly be holding a referenced, immutable or + // mutable, at the same time. + let mut maybe_inner = cell.borrow_mut(); + let inner = maybe_inner.get_or_insert_with(Inner::new); + + inner.registry.get_or_create_counter(key, |c| Counter::from_arc(c.clone())) + }) + } else { + self.inner.registry.get_or_create_counter(key, |c| Counter::from_arc(c.clone())) + } } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { let ckey = CompositeKey::new(MetricKind::Gauge, key.clone()); self.track_metric(ckey); - self.registry.get_or_create_gauge(key, |g| Gauge::from_arc(g.clone())) + + if self.is_per_thread { + PER_THREAD_INNER.with(move |cell| { + // Create the inner state if it doesn't yet exist. + // + // SAFETY: It's safe to use `borrow_mut` here, even though the parent method is `&self`, as this is a + // per-thread invocation, so no other caller could possibly be holding a referenced, immutable or + // mutable, at the same time. + let mut maybe_inner = cell.borrow_mut(); + let inner = maybe_inner.get_or_insert_with(Inner::new); + + inner.registry.get_or_create_gauge(key, |g| Gauge::from_arc(g.clone())) + }) + } else { + self.inner.registry.get_or_create_gauge(key, |g| Gauge::from_arc(g.clone())) + } } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { let ckey = CompositeKey::new(MetricKind::Histogram, key.clone()); self.track_metric(ckey); - self.registry.get_or_create_histogram(key, |h| Histogram::from_arc(h.clone())) + + if self.is_per_thread { + PER_THREAD_INNER.with(move |cell| { + // Create the inner state if it doesn't yet exist. + // + // SAFETY: It's safe to use `borrow_mut` here, even though the parent method is `&self`, as this is a + // per-thread invocation, so no other caller could possibly be holding a referenced, immutable or + // mutable, at the same time. + let mut maybe_inner = cell.borrow_mut(); + let inner = maybe_inner.get_or_insert_with(Inner::new); + + inner.registry.get_or_create_histogram(key, |h| Histogram::from_arc(h.clone())) + }) + } else { + self.inner.registry.get_or_create_histogram(key, |h| Histogram::from_arc(h.clone())) + } } } @@ -201,3 +361,64 @@ impl Default for DebuggingRecorder { DebuggingRecorder::new() } } + +#[cfg(test)] +mod tests { + use metrics::counter; + + use crate::{CompositeKey, MetricKind}; + + use super::{DebugValue, DebuggingRecorder, Snapshotter}; + + #[test] + fn per_thread() { + // Create the recorder in per-thread mode, get the snapshotter, and then spawn two threads that record some + // metrics. Neither thread should see the metrics of the other, and this main thread running the test should see + // _any_ metrics. + let recorder = DebuggingRecorder::per_thread(); + let snapshotter = recorder.snapshotter(); + + unsafe { metrics::clear_recorder() }; + recorder.install().expect("installing debugging recorder should not fail"); + + let t1 = std::thread::spawn(|| { + counter!("test_counter").increment(43); + + Snapshotter::current_thread_snapshot() + }); + + let t2 = std::thread::spawn(|| { + counter!("test_counter").increment(47); + + Snapshotter::current_thread_snapshot() + }); + + let t1_result = + t1.join().expect("thread 1 should not fail").expect("thread 1 should have metrics"); + let t2_result = + t2.join().expect("thread 2 should not fail").expect("thread 2 should have metrics"); + + let main_result = snapshotter.snapshot().into_vec(); + assert!(main_result.is_empty()); + + assert_eq!( + t1_result.into_vec(), + vec![( + CompositeKey::new(MetricKind::Counter, "test_counter".into()), + None, + None, + DebugValue::Counter(43), + )] + ); + + assert_eq!( + t2_result.into_vec(), + vec![( + CompositeKey::new(MetricKind::Counter, "test_counter".into()), + None, + None, + DebugValue::Counter(47), + )] + ); + } +} diff --git a/metrics-util/src/layers/fanout.rs b/metrics-util/src/layers/fanout.rs index 5d91e2b2..430b61d6 100644 --- a/metrics-util/src/layers/fanout.rs +++ b/metrics-util/src/layers/fanout.rs @@ -1,7 +1,8 @@ use std::sync::Arc; use metrics::{ - Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Recorder, Unit, + Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, Recorder, + SharedString, Unit, }; struct FanoutCounter { @@ -100,40 +101,47 @@ pub struct Fanout { } impl Recorder for Fanout { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString) { for recorder in &self.recorders { - recorder.describe_counter(key_name.clone(), unit, description); + recorder.describe_counter(key_name.clone(), unit, description.clone()); } } - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { for recorder in &self.recorders { - recorder.describe_gauge(key_name.clone(), unit, description); + recorder.describe_gauge(key_name.clone(), unit, description.clone()); } } - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString) { for recorder in &self.recorders { - recorder.describe_histogram(key_name.clone(), unit, description); + recorder.describe_histogram(key_name.clone(), unit, description.clone()); } } - fn register_counter(&self, key: &Key) -> Counter { - let counters = - self.recorders.iter().map(|recorder| recorder.register_counter(key)).collect(); + fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter { + let counters = self + .recorders + .iter() + .map(|recorder| recorder.register_counter(key, metadata)) + .collect(); FanoutCounter::from_counters(counters).into() } - fn register_gauge(&self, key: &Key) -> Gauge { - let gauges = self.recorders.iter().map(|recorder| recorder.register_gauge(key)).collect(); + fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge { + let gauges = + self.recorders.iter().map(|recorder| recorder.register_gauge(key, metadata)).collect(); FanoutGauge::from_gauges(gauges).into() } - fn register_histogram(&self, key: &Key) -> Histogram { - let histograms = - self.recorders.iter().map(|recorder| recorder.register_histogram(key)).collect(); + fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram { + let histograms = self + .recorders + .iter() + .map(|recorder| recorder.register_histogram(key, metadata)) + .collect(); FanoutHistogram::from_histograms(histograms).into() } @@ -169,23 +177,34 @@ mod tests { use crate::test_util::*; use metrics::{Counter, Gauge, Histogram, Unit}; + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); + #[test] fn test_basic_functionality() { let operations = vec![ RecorderOperation::DescribeCounter( "counter_key".into(), Some(Unit::Count), - "counter desc", + "counter desc".into(), + ), + RecorderOperation::DescribeGauge( + "gauge_key".into(), + Some(Unit::Bytes), + "gauge desc".into(), ), - RecorderOperation::DescribeGauge("gauge_key".into(), Some(Unit::Bytes), "gauge desc"), RecorderOperation::DescribeHistogram( "histogram_key".into(), Some(Unit::Nanoseconds), - "histogram desc", + "histogram desc".into(), + ), + RecorderOperation::RegisterCounter("counter_key".into(), Counter::noop(), &METADATA), + RecorderOperation::RegisterGauge("gauge_key".into(), Gauge::noop(), &METADATA), + RecorderOperation::RegisterHistogram( + "histogram_key".into(), + Histogram::noop(), + &METADATA, ), - RecorderOperation::RegisterCounter("counter_key".into(), Counter::noop()), - RecorderOperation::RegisterGauge("gauge_key".into(), Gauge::noop()), - RecorderOperation::RegisterHistogram("histogram_key".into(), Histogram::noop()), ]; let recorder1 = MockBasicRecorder::from_operations(operations.clone()); diff --git a/metrics-util/src/layers/filter.rs b/metrics-util/src/layers/filter.rs index bb7a698e..f7125605 100644 --- a/metrics-util/src/layers/filter.rs +++ b/metrics-util/src/layers/filter.rs @@ -1,6 +1,6 @@ use crate::layers::Layer; -use aho_corasick::{AhoCorasick, AhoCorasickBuilder}; -use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; +use aho_corasick::{AhoCorasick, AhoCorasickBuilder, AhoCorasickKind}; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; /// Filters and discards metrics matching certain name patterns. /// @@ -17,46 +17,46 @@ impl Filter { } impl Recorder for Filter { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString) { if self.should_filter(key_name.as_str()) { return; } self.inner.describe_counter(key_name, unit, description) } - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { if self.should_filter(key_name.as_str()) { return; } self.inner.describe_gauge(key_name, unit, description) } - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString) { if self.should_filter(key_name.as_str()) { return; } self.inner.describe_histogram(key_name, unit, description) } - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter { if self.should_filter(key.name()) { return Counter::noop(); } - self.inner.register_counter(key) + self.inner.register_counter(key, metadata) } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge { if self.should_filter(key.name()) { return Gauge::noop(); } - self.inner.register_gauge(key) + self.inner.register_gauge(key, metadata) } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram { if self.should_filter(key.name()) { return Histogram::noop(); } - self.inner.register_histogram(key) + self.inner.register_histogram(key, metadata) } } @@ -123,7 +123,7 @@ impl FilterLayer { /// searches from O(n + p) to O(n), where n is the length of the haystack. /// /// In general, it's a good idea to enable this if you're searching a small number of fairly - /// short patterns (~1000), or if you want the fastest possible search without regard to + /// short patterns, or if you want the fastest possible search without regard to /// compilation time or space usage. /// /// Defaults to `true`. @@ -140,9 +140,13 @@ impl Layer for FilterLayer { let mut automaton_builder = AhoCorasickBuilder::new(); let automaton = automaton_builder .ascii_case_insensitive(self.case_insensitive) - .dfa(self.use_dfa) - .auto_configure(&self.patterns) - .build(&self.patterns); + .kind(self.use_dfa.then(|| AhoCorasickKind::DFA)) + .build(&self.patterns) + // Documentation for `AhoCorasickBuilder::build` states that the error here will be + // related to exceeding some internal limits, but that those limits should generally be + // large enough for most use cases.. so I'm making the executive decision to consider + // that "good enough" and treat this as an exceptional error if it does occur. + .expect("should not fail to build filter automaton"); Filter { inner, automaton } } } @@ -153,59 +157,68 @@ mod tests { use crate::{layers::Layer, test_util::*}; use metrics::{Counter, Gauge, Histogram, Unit}; + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); + #[test] fn test_basic_functionality() { let inputs = vec![ RecorderOperation::DescribeCounter( "tokio.loops".into(), Some(Unit::Count), - "counter desc", + "counter desc".into(), ), RecorderOperation::DescribeGauge( "hyper.bytes_read".into(), Some(Unit::Bytes), - "gauge desc", + "gauge desc".into(), ), RecorderOperation::DescribeHistogram( "hyper.response_latency".into(), Some(Unit::Nanoseconds), - "histogram desc", + "histogram desc".into(), ), RecorderOperation::DescribeCounter( "tokio.spurious_wakeups".into(), Some(Unit::Count), - "counter desc", + "counter desc".into(), ), RecorderOperation::DescribeGauge( "bb8.pooled_conns".into(), Some(Unit::Count), - "gauge desc", + "gauge desc".into(), ), - RecorderOperation::RegisterCounter("tokio.loops".into(), Counter::noop()), - RecorderOperation::RegisterGauge("hyper.bytes_read".into(), Gauge::noop()), + RecorderOperation::RegisterCounter("tokio.loops".into(), Counter::noop(), &METADATA), + RecorderOperation::RegisterGauge("hyper.bytes_read".into(), Gauge::noop(), &METADATA), RecorderOperation::RegisterHistogram( "hyper.response_latency".into(), Histogram::noop(), + &METADATA, + ), + RecorderOperation::RegisterCounter( + "tokio.spurious_wakeups".into(), + Counter::noop(), + &METADATA, ), - RecorderOperation::RegisterCounter("tokio.spurious_wakeups".into(), Counter::noop()), - RecorderOperation::RegisterGauge("bb8.pooled_conns".into(), Gauge::noop()), + RecorderOperation::RegisterGauge("bb8.pooled_conns".into(), Gauge::noop(), &METADATA), ]; let expectations = vec![ RecorderOperation::DescribeGauge( "hyper.bytes_read".into(), Some(Unit::Bytes), - "gauge desc", + "gauge desc".into(), ), RecorderOperation::DescribeHistogram( "hyper.response_latency".into(), Some(Unit::Nanoseconds), - "histogram desc", + "histogram desc".into(), ), - RecorderOperation::RegisterGauge("hyper.bytes_read".into(), Gauge::noop()), + RecorderOperation::RegisterGauge("hyper.bytes_read".into(), Gauge::noop(), &METADATA), RecorderOperation::RegisterHistogram( "hyper.response_latency".into(), Histogram::noop(), + &METADATA, ), ]; @@ -224,53 +237,59 @@ mod tests { RecorderOperation::DescribeCounter( "tokiO.loops".into(), Some(Unit::Count), - "counter desc", + "counter desc".into(), ), RecorderOperation::DescribeGauge( "hyper.bytes_read".into(), Some(Unit::Bytes), - "gauge desc", + "gauge desc".into(), ), RecorderOperation::DescribeHistogram( "hyper.response_latency".into(), Some(Unit::Nanoseconds), - "histogram desc", + "histogram desc".into(), ), RecorderOperation::DescribeCounter( "Tokio.spurious_wakeups".into(), Some(Unit::Count), - "counter desc", + "counter desc".into(), ), RecorderOperation::DescribeGauge( "bB8.pooled_conns".into(), Some(Unit::Count), - "gauge desc", + "gauge desc".into(), ), - RecorderOperation::RegisterCounter("tokiO.loops".into(), Counter::noop()), - RecorderOperation::RegisterGauge("hyper.bytes_read".into(), Gauge::noop()), + RecorderOperation::RegisterCounter("tokiO.loops".into(), Counter::noop(), &METADATA), + RecorderOperation::RegisterGauge("hyper.bytes_read".into(), Gauge::noop(), &METADATA), RecorderOperation::RegisterHistogram( "hyper.response_latency".into(), Histogram::noop(), + &METADATA, + ), + RecorderOperation::RegisterCounter( + "Tokio.spurious_wakeups".into(), + Counter::noop(), + &METADATA, ), - RecorderOperation::RegisterCounter("Tokio.spurious_wakeups".into(), Counter::noop()), - RecorderOperation::RegisterGauge("bB8.pooled_conns".into(), Gauge::noop()), + RecorderOperation::RegisterGauge("bB8.pooled_conns".into(), Gauge::noop(), &METADATA), ]; let expectations = vec![ RecorderOperation::DescribeGauge( "hyper.bytes_read".into(), Some(Unit::Bytes), - "gauge desc", + "gauge desc".into(), ), RecorderOperation::DescribeHistogram( "hyper.response_latency".into(), Some(Unit::Nanoseconds), - "histogram desc", + "histogram desc".into(), ), - RecorderOperation::RegisterGauge("hyper.bytes_read".into(), Gauge::noop()), + RecorderOperation::RegisterGauge("hyper.bytes_read".into(), Gauge::noop(), &METADATA), RecorderOperation::RegisterHistogram( "hyper.response_latency".into(), Histogram::noop(), + &METADATA, ), ]; diff --git a/metrics-util/src/layers/mod.rs b/metrics-util/src/layers/mod.rs index 231093fb..bddafc5d 100644 --- a/metrics-util/src/layers/mod.rs +++ b/metrics-util/src/layers/mod.rs @@ -8,7 +8,7 @@ //! Here's an example of a layer that filters out all metrics that start with a specific string: //! //! ```rust -//! # use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; +//! # use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; //! # use metrics::NoopRecorder as BasicRecorder; //! # use metrics_util::layers::{Layer, Stack, PrefixLayer}; //! // A simple layer that denies any metrics that have "stairway" or "heaven" in their name. @@ -26,7 +26,7 @@ //! &self, //! key_name: KeyName, //! unit: Option, -//! description: &'static str, +//! description: SharedString, //! ) { //! if self.is_invalid_key(key_name.as_str()) { //! return; @@ -34,7 +34,7 @@ //! self.0.describe_counter(key_name, unit, description) //! } //! -//! fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { +//! fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { //! if self.is_invalid_key(key_name.as_str()) { //! return; //! } @@ -45,7 +45,7 @@ //! &self, //! key_name: KeyName, //! unit: Option, -//! description: &'static str, +//! description: SharedString, //! ) { //! if self.is_invalid_key(key_name.as_str()) { //! return; @@ -53,25 +53,25 @@ //! self.0.describe_histogram(key_name, unit, description) //! } //! -//! fn register_counter(&self, key: &Key) -> Counter { +//! fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter { //! if self.is_invalid_key(key.name()) { //! return Counter::noop(); //! } -//! self.0.register_counter(key) +//! self.0.register_counter(key, metadata) //! } //! -//! fn register_gauge(&self, key: &Key) -> Gauge { +//! fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge { //! if self.is_invalid_key(key.name()) { //! return Gauge::noop(); //! } -//! self.0.register_gauge(key) +//! self.0.register_gauge(key, metadata) //! } //! -//! fn register_histogram(&self, key: &Key) -> Histogram { +//! fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram { //! if self.is_invalid_key(key.name()) { //! return Histogram::noop(); //! } -//! self.0.register_histogram(key) +//! self.0.register_histogram(key, metadata) //! } //! } //! @@ -94,13 +94,13 @@ //! let layered = layer.layer(recorder); //! metrics::set_boxed_recorder(Box::new(layered)).expect("failed to install recorder"); //! -//! # metrics::clear_recorder(); +//! # unsafe { metrics::clear_recorder() }; //! //! // Working with layers directly is a bit cumbersome, though, so let's use a `Stack`. //! let stack = Stack::new(BasicRecorder); //! stack.push(StairwayDenyLayer::default()).install().expect("failed to install stack"); //! -//! # metrics::clear_recorder(); +//! # unsafe { metrics::clear_recorder() }; //! //! // `Stack` makes it easy to chain layers together, as well. //! let stack = Stack::new(BasicRecorder); @@ -111,7 +111,7 @@ //! .expect("failed to install stack"); //! # } //! ``` -use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; use metrics::SetRecorderError; @@ -167,27 +167,27 @@ impl Stack { } impl Recorder for Stack { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString) { self.inner.describe_counter(key_name, unit, description); } - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { self.inner.describe_gauge(key_name, unit, description); } - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString) { self.inner.describe_histogram(key_name, unit, description); } - fn register_counter(&self, key: &Key) -> Counter { - self.inner.register_counter(key) + fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter { + self.inner.register_counter(key, metadata) } - fn register_gauge(&self, key: &Key) -> Gauge { - self.inner.register_gauge(key) + fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge { + self.inner.register_gauge(key, metadata) } - fn register_histogram(&self, key: &Key) -> Histogram { - self.inner.register_histogram(key) + fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram { + self.inner.register_histogram(key, metadata) } } diff --git a/metrics-util/src/layers/prefix.rs b/metrics-util/src/layers/prefix.rs index 440733ba..6c4f3829 100644 --- a/metrics-util/src/layers/prefix.rs +++ b/metrics-util/src/layers/prefix.rs @@ -1,5 +1,5 @@ use crate::layers::Layer; -use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, SharedString, Unit}; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; /// Applies a prefix to every metric key. /// @@ -30,34 +30,34 @@ impl Prefix { } impl Recorder for Prefix { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString) { let new_key_name = self.prefix_key_name(key_name); self.inner.describe_counter(new_key_name, unit, description) } - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { let new_key_name = self.prefix_key_name(key_name); self.inner.describe_gauge(new_key_name, unit, description) } - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString) { let new_key_name = self.prefix_key_name(key_name); self.inner.describe_histogram(new_key_name, unit, description) } - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter { let new_key = self.prefix_key(key); - self.inner.register_counter(&new_key) + self.inner.register_counter(&new_key, metadata) } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge { let new_key = self.prefix_key(key); - self.inner.register_gauge(&new_key) + self.inner.register_gauge(&new_key, metadata) } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram { let new_key = self.prefix_key(key); - self.inner.register_histogram(&new_key) + self.inner.register_histogram(&new_key, metadata) } } @@ -88,44 +88,63 @@ mod tests { use crate::test_util::*; use metrics::{Counter, Gauge, Histogram, Key, KeyName, Unit}; + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); + #[test] fn test_basic_functionality() { let inputs = vec![ RecorderOperation::DescribeCounter( "counter_key".into(), Some(Unit::Count), - "counter desc", + "counter desc".into(), + ), + RecorderOperation::DescribeGauge( + "gauge_key".into(), + Some(Unit::Bytes), + "gauge desc".into(), ), - RecorderOperation::DescribeGauge("gauge_key".into(), Some(Unit::Bytes), "gauge desc"), RecorderOperation::DescribeHistogram( "histogram_key".into(), Some(Unit::Nanoseconds), - "histogram desc", + "histogram desc".into(), + ), + RecorderOperation::RegisterCounter("counter_key".into(), Counter::noop(), &METADATA), + RecorderOperation::RegisterGauge("gauge_key".into(), Gauge::noop(), &METADATA), + RecorderOperation::RegisterHistogram( + "histogram_key".into(), + Histogram::noop(), + &METADATA, ), - RecorderOperation::RegisterCounter("counter_key".into(), Counter::noop()), - RecorderOperation::RegisterGauge("gauge_key".into(), Gauge::noop()), - RecorderOperation::RegisterHistogram("histogram_key".into(), Histogram::noop()), ]; let expectations = vec![ RecorderOperation::DescribeCounter( "testing.counter_key".into(), Some(Unit::Count), - "counter desc", + "counter desc".into(), ), RecorderOperation::DescribeGauge( "testing.gauge_key".into(), Some(Unit::Bytes), - "gauge desc", + "gauge desc".into(), ), RecorderOperation::DescribeHistogram( "testing.histogram_key".into(), Some(Unit::Nanoseconds), - "histogram desc", + "histogram desc".into(), + ), + RecorderOperation::RegisterCounter( + "testing.counter_key".into(), + Counter::noop(), + &METADATA, + ), + RecorderOperation::RegisterGauge("testing.gauge_key".into(), Gauge::noop(), &METADATA), + RecorderOperation::RegisterHistogram( + "testing.histogram_key".into(), + Histogram::noop(), + &METADATA, ), - RecorderOperation::RegisterCounter("testing.counter_key".into(), Counter::noop()), - RecorderOperation::RegisterGauge("testing.gauge_key".into(), Gauge::noop()), - RecorderOperation::RegisterHistogram("testing.histogram_key".into(), Histogram::noop()), ]; let recorder = MockBasicRecorder::from_operations(expectations); diff --git a/metrics-util/src/layers/router.rs b/metrics-util/src/layers/router.rs index a9b11eae..867049d1 100644 --- a/metrics-util/src/layers/router.rs +++ b/metrics-util/src/layers/router.rs @@ -1,4 +1,4 @@ -use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; use radix_trie::{Trie, TrieCommon}; use crate::{MetricKind, MetricKindMask}; @@ -39,34 +39,34 @@ impl Router { } impl Recorder for Router { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString) { let target = self.route(MetricKind::Counter, key_name.as_str(), &self.counter_routes); target.describe_counter(key_name, unit, description) } - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { let target = self.route(MetricKind::Gauge, key_name.as_str(), &self.gauge_routes); target.describe_gauge(key_name, unit, description) } - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString) { let target = self.route(MetricKind::Histogram, key_name.as_str(), &self.histogram_routes); target.describe_histogram(key_name, unit, description) } - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter { let target = self.route(MetricKind::Counter, key.name(), &self.counter_routes); - target.register_counter(key) + target.register_counter(key, metadata) } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge { let target = self.route(MetricKind::Gauge, key.name(), &self.gauge_routes); - target.register_gauge(key) + target.register_gauge(key, metadata) } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram { let target = self.route(MetricKind::Histogram, key.name(), &self.histogram_routes); - target.register_histogram(key) + target.register_histogram(key, metadata) } } @@ -161,24 +161,30 @@ impl RouterBuilder { #[cfg(test)] mod tests { - use mockall::{mock, predicate::eq, Sequence}; + use mockall::{ + mock, + predicate::{always, eq}, + Sequence, + }; use std::borrow::Cow; use super::RouterBuilder; use crate::MetricKindMask; - use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; + use metrics::{ + Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit, + }; mock! { pub TestRecorder { } impl Recorder for TestRecorder { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str); - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str); - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str); - fn register_counter(&self, key: &Key) -> Counter; - fn register_gauge(&self, key: &Key) -> Gauge; - fn register_histogram(&self, key: &Key) -> Histogram; + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString); + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString); + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString); + fn register_counter<'a>(&'a self, key: &'a Key, metadata: &'a Metadata<'a>) -> Counter; + fn register_gauge<'a>(&'a self, key: &'a Key, metadata: &'a Metadata<'a>) -> Gauge; + fn register_histogram<'a>(&'a self, key: &'a Key, metadata: &'a Metadata<'a>) -> Histogram; } } @@ -215,33 +221,36 @@ mod tests { let mut seq = Sequence::new(); + static METADATA: metrics::Metadata = + metrics::Metadata::new(module_path!(), metrics::Level::INFO, Some(module_path!())); + default_mock .expect_register_counter() .times(1) .in_sequence(&mut seq) - .with(eq(default_counter.clone())) - .returning(|_| Counter::noop()); + .with(eq(default_counter.clone()), always()) + .returning(|_, _| Counter::noop()); counter_mock .expect_register_counter() .times(1) .in_sequence(&mut seq) - .with(eq(override_counter.clone())) - .returning(|_| Counter::noop()); + .with(eq(override_counter.clone()), always()) + .returning(|_, _| Counter::noop()); all_mock .expect_register_counter() .times(1) .in_sequence(&mut seq) - .with(eq(all_override.clone())) - .returning(|_| Counter::noop()); + .with(eq(all_override.clone()), always()) + .returning(|_, _| Counter::noop()); all_mock .expect_register_histogram() .times(1) .in_sequence(&mut seq) - .with(eq(all_override.clone())) - .returning(|_| Histogram::noop()); + .with(eq(all_override.clone()), always()) + .returning(|_, _| Histogram::noop()); let mut builder = RouterBuilder::from_recorder(default_mock); builder.add_route(MetricKindMask::COUNTER, "counter_override", counter_mock).add_route( @@ -251,9 +260,9 @@ mod tests { ); let recorder = builder.build(); - let _ = recorder.register_counter(&default_counter); - let _ = recorder.register_counter(&override_counter); - let _ = recorder.register_counter(&all_override); - let _ = recorder.register_histogram(&all_override); + let _ = recorder.register_counter(&default_counter, &METADATA); + let _ = recorder.register_counter(&override_counter, &METADATA); + let _ = recorder.register_counter(&all_override, &METADATA); + let _ = recorder.register_histogram(&all_override, &METADATA); } } diff --git a/metrics-util/src/lib.rs b/metrics-util/src/lib.rs index 42641fd7..c3052858 100644 --- a/metrics-util/src/lib.rs +++ b/metrics-util/src/lib.rs @@ -31,6 +31,9 @@ pub use kind::{MetricKind, MetricKindMask}; mod histogram; pub use histogram::Histogram; +mod recoverable; +pub use recoverable::RecoverableRecorder; + #[cfg(feature = "summary")] mod summary; #[cfg(feature = "summary")] diff --git a/metrics-util/src/recoverable.rs b/metrics-util/src/recoverable.rs new file mode 100644 index 00000000..9fb31236 --- /dev/null +++ b/metrics-util/src/recoverable.rs @@ -0,0 +1,321 @@ +use std::sync::{Arc, Weak}; + +use metrics::{ + Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SetRecorderError, SharedString, + Unit, +}; + +/// Wraps a recorder to allow for recovering it after being installed. +/// +/// Installing a recorder generally involves providing an owned value, which means that it is not +/// possible to recover the recorder after it has been installed. For some recorder implementations, +/// it can be important to perform finalization before the application exits, which is not possible +/// if the application cannot consume the recorder. +/// +/// `RecoverableRecorder` allows wrapping a recorder such that a weak reference to it is installed +/// globally, while the recorder itself is held by `RecoverableRecorder`. This allows for recovering +/// the recorder whenever the application chooses. +/// +/// ## As a drop guard +/// +/// While `RecoverableRecorder` provides a method to manually recover the recorder directly, one +/// particular benefit is that due to how the recorder is wrapped, when `RecoverableRecorder` is +/// dropped, and the last active reference to it is dropped, the recorder itself will be dropped. +/// +/// This allows using `RecoverableRecorder` as a drop guard, ensuring that by dropping it, the +/// recorder itself will be dropped, and any finalization logic implemented for the recorder will be +/// run. +pub struct RecoverableRecorder { + recorder: Arc, +} + +impl RecoverableRecorder { + /// Creates a new `RecoverableRecorder`, wrapping the given recorder. + /// + /// A weakly-referenced version of the recorder is installed globally, while the original + /// recorder is held within `RecoverableRecorder`, and can be recovered by calling `into_inner`. + /// + /// # Errors + /// + /// If a recorder is already installed, an error is returned. + pub fn from_recorder(recorder: R) -> Result { + let recorder = Arc::new(recorder); + + let wrapped = WeakRecorder::from_arc(&recorder); + metrics::set_boxed_recorder(Box::new(wrapped))?; + + Ok(Self { recorder }) + } + + /// Consumes this wrapper, returning the original recorder. + /// + /// This method will loop until there are no active weak references to the recorder. It is not + /// advised to call this method under heavy load, as doing so is not deterministic or ordered + /// and may block for an indefinite amount of time. + pub fn into_inner(mut self) -> R { + loop { + match Arc::try_unwrap(self.recorder) { + Ok(recorder) => break recorder, + Err(recorder) => { + self.recorder = recorder; + } + } + } + } +} + +struct WeakRecorder { + recorder: Weak, +} + +impl WeakRecorder { + fn from_arc(recorder: &Arc) -> Self { + Self { recorder: Arc::downgrade(recorder) } + } +} + +impl Recorder for WeakRecorder { + fn describe_counter(&self, key: KeyName, unit: Option, description: SharedString) { + if let Some(recorder) = self.recorder.upgrade() { + recorder.describe_counter(key, unit, description); + } + } + + fn describe_gauge(&self, key: KeyName, unit: Option, description: SharedString) { + if let Some(recorder) = self.recorder.upgrade() { + recorder.describe_gauge(key, unit, description); + } + } + + fn describe_histogram(&self, key: KeyName, unit: Option, description: SharedString) { + if let Some(recorder) = self.recorder.upgrade() { + recorder.describe_histogram(key, unit, description); + } + } + + fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter { + if let Some(recorder) = self.recorder.upgrade() { + recorder.register_counter(key, metadata) + } else { + Counter::noop() + } + } + + fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge { + if let Some(recorder) = self.recorder.upgrade() { + recorder.register_gauge(key, metadata) + } else { + Gauge::noop() + } + } + + fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram { + if let Some(recorder) = self.recorder.upgrade() { + recorder.register_histogram(key, metadata) + } else { + Histogram::noop() + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; + + use super::*; + use metrics::{atomics::AtomicU64, CounterFn, GaugeFn, HistogramFn, Key, Recorder}; + + struct CounterWrapper(AtomicU64); + struct GaugeWrapper(AtomicU64); + struct HistogramWrapper(AtomicU64); + + impl CounterWrapper { + fn get(&self) -> u64 { + self.0.load(Ordering::Acquire) + } + } + + impl GaugeWrapper { + fn get(&self) -> u64 { + self.0.load(Ordering::Acquire) + } + } + + impl HistogramWrapper { + fn get(&self) -> u64 { + self.0.load(Ordering::Acquire) + } + } + + impl CounterFn for CounterWrapper { + fn increment(&self, value: u64) { + self.0.fetch_add(value, Ordering::Release); + } + + fn absolute(&self, value: u64) { + self.0.store(value, Ordering::Release); + } + } + + impl GaugeFn for GaugeWrapper { + fn increment(&self, value: f64) { + self.0.fetch_add(value as u64, Ordering::Release); + } + + fn decrement(&self, value: f64) { + self.0.fetch_sub(value as u64, Ordering::Release); + } + + fn set(&self, value: f64) { + self.0.store(value as u64, Ordering::Release); + } + } + + impl HistogramFn for HistogramWrapper { + fn record(&self, value: f64) { + self.0.fetch_add(value as u64, Ordering::Release); + } + } + + struct TestRecorder { + dropped: Arc, + counter: Arc, + gauge: Arc, + histogram: Arc, + } + + impl TestRecorder { + fn new() -> (Self, Arc, Arc, Arc) { + let (recorder, _, counter, gauge, histogram) = Self::new_with_drop(); + (recorder, counter, gauge, histogram) + } + + fn new_with_drop( + ) -> (Self, Arc, Arc, Arc, Arc) + { + let dropped = Arc::new(AtomicBool::new(false)); + let counter = Arc::new(CounterWrapper(AtomicU64::new(0))); + let gauge = Arc::new(GaugeWrapper(AtomicU64::new(0))); + let histogram = Arc::new(HistogramWrapper(AtomicU64::new(0))); + + let recorder = Self { + dropped: Arc::clone(&dropped), + counter: Arc::clone(&counter), + gauge: Arc::clone(&gauge), + histogram: Arc::clone(&histogram), + }; + + (recorder, dropped, counter, gauge, histogram) + } + } + + impl Recorder for TestRecorder { + fn describe_counter(&self, _key: KeyName, _unit: Option, _description: SharedString) { + todo!() + } + + fn describe_gauge(&self, _key: KeyName, _unit: Option, _description: SharedString) { + todo!() + } + + fn describe_histogram( + &self, + _key: KeyName, + _unit: Option, + _description: SharedString, + ) { + todo!() + } + + fn register_counter(&self, _: &Key, _: &Metadata<'_>) -> Counter { + Counter::from_arc(Arc::clone(&self.counter)) + } + + fn register_gauge(&self, _: &Key, _: &Metadata<'_>) -> Gauge { + Gauge::from_arc(Arc::clone(&self.gauge)) + } + + fn register_histogram(&self, _: &Key, _: &Metadata<'_>) -> Histogram { + Histogram::from_arc(Arc::clone(&self.histogram)) + } + } + + impl Drop for TestRecorder { + fn drop(&mut self) { + self.dropped.store(true, Ordering::Release); + } + } + + #[test] + fn basic() { + // Create and install the recorder. + let (recorder, counter, gauge, histogram) = TestRecorder::new(); + unsafe { + metrics::clear_recorder(); + } + let recoverable = + RecoverableRecorder::from_recorder(recorder).expect("failed to install recorder"); + + // Record some metrics, and make sure the atomics for each metric type are + // incremented as we would expect them to be. + metrics::counter!("counter").increment(5); + metrics::gauge!("gauge").increment(5.0); + metrics::gauge!("gauge").increment(5.0); + metrics::histogram!("histogram").record(5.0); + metrics::histogram!("histogram").record(5.0); + metrics::histogram!("histogram").record(5.0); + + let _recorder = recoverable.into_inner(); + assert_eq!(counter.get(), 5); + assert_eq!(gauge.get(), 10); + assert_eq!(histogram.get(), 15); + + // Now that we've recovered the recorder, incrementing the same metrics should + // not actually increment the value of the atomics for each metric type. + metrics::counter!("counter").increment(7); + metrics::gauge!("gauge").increment(7.0); + metrics::histogram!("histogram").record(7.0); + + assert_eq!(counter.get(), 5); + assert_eq!(gauge.get(), 10); + assert_eq!(histogram.get(), 15); + } + + #[test] + fn on_drop() { + // Create and install the recorder. + let (recorder, dropped, counter, gauge, histogram) = TestRecorder::new_with_drop(); + unsafe { + metrics::clear_recorder(); + } + let recoverable = + RecoverableRecorder::from_recorder(recorder).expect("failed to install recorder"); + + // Record some metrics, and make sure the atomics for each metric type are + // incremented as we would expect them to be. + metrics::counter!("counter").increment(5); + metrics::gauge!("gauge").increment(5.0); + metrics::gauge!("gauge").increment(5.0); + metrics::histogram!("histogram").record(5.0); + metrics::histogram!("histogram").record(5.0); + metrics::histogram!("histogram").record(5.0); + + drop(recoverable.into_inner()); + assert_eq!(counter.get(), 5); + assert_eq!(gauge.get(), 10); + assert_eq!(histogram.get(), 15); + + // Now that we've recovered the recorder, incrementing the same metrics should + // not actually increment the value of the atomics for each metric type. + metrics::counter!("counter").increment(7); + metrics::gauge!("gauge").increment(7.0); + metrics::histogram!("histogram").record(7.0); + + assert_eq!(counter.get(), 5); + assert_eq!(gauge.get(), 10); + assert_eq!(histogram.get(), 15); + + // And we should be able to check that the recorder was indeed dropped. + assert!(dropped.load(Ordering::Acquire)); + } +} diff --git a/metrics-util/src/registry/mod.rs b/metrics-util/src/registry/mod.rs index b1dd42bb..68a0e5f1 100644 --- a/metrics-util/src/registry/mod.rs +++ b/metrics-util/src/registry/mod.rs @@ -1,11 +1,14 @@ //! High-performance metrics storage. mod storage; -use std::{hash::BuildHasherDefault, iter::repeat}; +use std::{ + hash::BuildHasherDefault, + iter::repeat, + sync::{PoisonError, RwLock}, +}; use hashbrown::{hash_map::RawEntryMut, HashMap}; use metrics::{Key, KeyHasher}; -use parking_lot::RwLock; pub use storage::{AtomicStorage, Storage}; #[cfg(feature = "recency")] @@ -29,17 +32,14 @@ type RegistryHashMap = HashMap>; /// ## Using `Registry` as the basis of an exporter /// /// As a reusable building blocking for building exporter implementations, users should look at -/// [`Key`] and [`AtomicStorage`][crate::registry::AtomicStorage] to use for their key and storage, -/// respectively. +/// [`Key`] and [`AtomicStorage`] to use for their key and storage, respectively. /// /// These two implementations provide behavior that is suitable for most exporters, providing /// seamless integration with the existing key type used by the core /// [`Recorder`][metrics::Recorder] trait, as well as atomic storage for metrics. /// -/// In some cases, users may prefer -/// [`GenerationalAtomicStorage`][crate::registry::GenerationalAtomicStorage] when know if a metric -/// has been touched, even if its value has not changed since the last time it was observed, is -/// necessary. +/// In some cases, users may prefer [`GenerationalAtomicStorage`] when know if a metric has been +/// touched, even if its value has not changed since the last time it was observed, is necessary. /// /// ## Performance /// @@ -143,13 +143,13 @@ where /// does not ensure that callers will see the registry as entirely empty at any given point. pub fn clear(&self) { for shard in &self.counters { - shard.write().clear(); + shard.write().unwrap_or_else(PoisonError::into_inner).clear(); } for shard in &self.gauges { - shard.write().clear(); + shard.write().unwrap_or_else(PoisonError::into_inner).clear(); } for shard in &self.histograms { - shard.write().clear(); + shard.write().unwrap_or_else(PoisonError::into_inner).clear(); } } @@ -164,13 +164,13 @@ where let (hash, shard) = self.get_hash_and_shard_for_counter(key); // Try and get the handle if it exists, running our operation if we succeed. - let shard_read = shard.read(); + let shard_read = shard.read().unwrap_or_else(PoisonError::into_inner); if let Some((_, v)) = shard_read.raw_entry().from_key_hashed_nocheck(hash, key) { op(v) } else { // Switch to write guard and insert the handle first. drop(shard_read); - let mut shard_write = shard.write(); + let mut shard_write = shard.write().unwrap_or_else(PoisonError::into_inner); let v = if let Some((_, v)) = shard_write.raw_entry().from_key_hashed_nocheck(hash, key) { v @@ -198,13 +198,13 @@ where let (hash, shard) = self.get_hash_and_shard_for_gauge(key); // Try and get the handle if it exists, running our operation if we succeed. - let shard_read = shard.read(); + let shard_read = shard.read().unwrap_or_else(PoisonError::into_inner); if let Some((_, v)) = shard_read.raw_entry().from_key_hashed_nocheck(hash, key) { op(v) } else { // Switch to write guard and insert the handle first. drop(shard_read); - let mut shard_write = shard.write(); + let mut shard_write = shard.write().unwrap_or_else(PoisonError::into_inner); let v = if let Some((_, v)) = shard_write.raw_entry().from_key_hashed_nocheck(hash, key) { v @@ -232,13 +232,13 @@ where let (hash, shard) = self.get_hash_and_shard_for_histogram(key); // Try and get the handle if it exists, running our operation if we succeed. - let shard_read = shard.read(); + let shard_read = shard.read().unwrap_or_else(PoisonError::into_inner); if let Some((_, v)) = shard_read.raw_entry().from_key_hashed_nocheck(hash, key) { op(v) } else { // Switch to write guard and insert the handle first. drop(shard_read); - let mut shard_write = shard.write(); + let mut shard_write = shard.write().unwrap_or_else(PoisonError::into_inner); let v = if let Some((_, v)) = shard_write.raw_entry().from_key_hashed_nocheck(hash, key) { v @@ -260,7 +260,7 @@ where /// Returns `true` if the counter existed and was removed, `false` otherwise. pub fn delete_counter(&self, key: &K) -> bool { let (hash, shard) = self.get_hash_and_shard_for_counter(key); - let mut shard_write = shard.write(); + let mut shard_write = shard.write().unwrap_or_else(PoisonError::into_inner); let entry = shard_write.raw_entry_mut().from_key_hashed_nocheck(hash, key); if let RawEntryMut::Occupied(entry) = entry { let _ = entry.remove_entry(); @@ -275,7 +275,7 @@ where /// Returns `true` if the gauge existed and was removed, `false` otherwise. pub fn delete_gauge(&self, key: &K) -> bool { let (hash, shard) = self.get_hash_and_shard_for_gauge(key); - let mut shard_write = shard.write(); + let mut shard_write = shard.write().unwrap_or_else(PoisonError::into_inner); let entry = shard_write.raw_entry_mut().from_key_hashed_nocheck(hash, key); if let RawEntryMut::Occupied(entry) = entry { let _ = entry.remove_entry(); @@ -290,7 +290,7 @@ where /// Returns `true` if the histogram existed and was removed, `false` otherwise. pub fn delete_histogram(&self, key: &K) -> bool { let (hash, shard) = self.get_hash_and_shard_for_histogram(key); - let mut shard_write = shard.write(); + let mut shard_write = shard.write().unwrap_or_else(PoisonError::into_inner); let entry = shard_write.raw_entry_mut().from_key_hashed_nocheck(hash, key); if let RawEntryMut::Occupied(entry) = entry { let _ = entry.remove_entry(); @@ -312,7 +312,7 @@ where F: FnMut(&K, &S::Counter), { for subshard in self.counters.iter() { - let shard_read = subshard.read(); + let shard_read = subshard.read().unwrap_or_else(PoisonError::into_inner); for (key, counter) in shard_read.iter() { collect(key, counter); } @@ -331,7 +331,7 @@ where F: FnMut(&K, &S::Gauge), { for subshard in self.gauges.iter() { - let shard_read = subshard.read(); + let shard_read = subshard.read().unwrap_or_else(PoisonError::into_inner); for (key, gauge) in shard_read.iter() { collect(key, gauge); } @@ -350,7 +350,7 @@ where F: FnMut(&K, &S::Histogram), { for subshard in self.histograms.iter() { - let shard_read = subshard.read(); + let shard_read = subshard.read().unwrap_or_else(PoisonError::into_inner); for (key, histogram) in shard_read.iter() { collect(key, histogram); } @@ -393,8 +393,7 @@ where #[cfg(test)] mod tests { - use atomic_shim::AtomicU64; - use metrics::{CounterFn, Key}; + use metrics::{atomics::AtomicU64, CounterFn, Key}; use super::Registry; use std::sync::{atomic::Ordering, Arc}; diff --git a/metrics-util/src/registry/recency.rs b/metrics-util/src/registry/recency.rs index aeb15b46..0ccf014a 100644 --- a/metrics-util/src/registry/recency.rs +++ b/metrics-util/src/registry/recency.rs @@ -23,12 +23,11 @@ //! observed, to build a complete picture that allows deciding if a given metric has gone "idle" or //! not, and thus whether it should actually be deleted. use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use std::{collections::HashMap, ops::DerefMut}; use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn}; -use parking_lot::Mutex; use quanta::{Clock, Instant}; use crate::Hashable; @@ -248,12 +247,15 @@ where /// method will return `true` and will update the last update time internally. If the given key /// has not been updated recently enough, the key will be removed from the given registry if the /// given generation also matches. - pub fn should_store_counter( + pub fn should_store_counter( &self, key: &K, gen: Generation, - registry: &Registry, - ) -> bool { + registry: &Registry, + ) -> bool + where + S: Storage, + { self.should_store(key, gen, registry, MetricKind::Counter, |registry, key| { registry.delete_counter(key) }) @@ -265,12 +267,10 @@ where /// method will return `true` and will update the last update time internally. If the given key /// has not been updated recently enough, the key will be removed from the given registry if the /// given generation also matches. - pub fn should_store_gauge( - &self, - key: &K, - gen: Generation, - registry: &Registry, - ) -> bool { + pub fn should_store_gauge(&self, key: &K, gen: Generation, registry: &Registry) -> bool + where + S: Storage, + { self.should_store(key, gen, registry, MetricKind::Gauge, |registry, key| { registry.delete_gauge(key) }) @@ -282,31 +282,35 @@ where /// method will return `true` and will update the last update time internally. If the given key /// has not been updated recently enough, the key will be removed from the given registry if the /// given generation also matches. - pub fn should_store_histogram( + pub fn should_store_histogram( &self, key: &K, gen: Generation, - registry: &Registry, - ) -> bool { + registry: &Registry, + ) -> bool + where + S: Storage, + { self.should_store(key, gen, registry, MetricKind::Histogram, |registry, key| { registry.delete_histogram(key) }) } - fn should_store( + fn should_store( &self, key: &K, gen: Generation, - registry: &Registry, + registry: &Registry, kind: MetricKind, delete_op: F, ) -> bool where - F: Fn(&Registry, &K) -> bool, + F: Fn(&Registry, &K) -> bool, + S: Storage, { if let Some(idle_timeout) = self.idle_timeout { if self.mask.matches(kind) { - let mut guard = self.inner.lock(); + let mut guard = self.inner.lock().unwrap_or_else(PoisonError::into_inner); let (clock, entries) = guard.deref_mut(); let now = clock.now(); diff --git a/metrics-util/src/registry/storage.rs b/metrics-util/src/registry/storage.rs index e115c872..0e61cf9a 100644 --- a/metrics-util/src/registry/storage.rs +++ b/metrics-util/src/registry/storage.rs @@ -1,7 +1,6 @@ use std::sync::Arc; -use atomic_shim::AtomicU64; -use metrics::{CounterFn, GaugeFn, HistogramFn}; +use metrics::{atomics::AtomicU64, CounterFn, GaugeFn, HistogramFn}; use crate::AtomicBucket; diff --git a/metrics-util/src/summary.rs b/metrics-util/src/summary.rs index 85dbf2f5..cd4c5cc2 100644 --- a/metrics-util/src/summary.rs +++ b/metrics-util/src/summary.rs @@ -1,4 +1,5 @@ use sketches_ddsketch::{Config, DDSketch}; +use std::fmt; /// A quantile sketch with relative-error guarantees. /// @@ -41,12 +42,7 @@ use sketches_ddsketch::{Config, DDSketch}; /// [hdrhistogram]: https://docs.rs/hdrhistogram #[derive(Clone)] pub struct Summary { - negative: DDSketch, - positive: DDSketch, - min_value: f64, - zeroes: usize, - min: f64, - max: f64, + sketch: DDSketch, } impl Summary { @@ -68,14 +64,7 @@ impl Summary { pub fn new(alpha: f64, max_buckets: u32, min_value: f64) -> Summary { let config = Config::new(alpha, max_buckets, min_value.abs()); - Summary { - negative: DDSketch::new(config), - positive: DDSketch::new(config), - min_value: min_value.abs(), - zeroes: 0, - min: f64::INFINITY, - max: f64::NEG_INFINITY, - } + Summary { sketch: DDSketch::new(config) } } /// Creates a new [`Summary`] with default values. @@ -100,21 +89,7 @@ impl Summary { return; } - if value < self.min { - self.min = value; - } - - if value > self.max { - self.max = value; - } - - if value > self.min_value { - self.positive.add(value); - } else if value < -self.min_value { - self.negative.add(-value); - } else { - self.zeroes += 1; - } + self.sketch.add(value); } /// Gets the estimated value at the given quantile. @@ -122,41 +97,35 @@ impl Summary { /// If the sketch is empty, or if the quantile is less than 0.0 or greater than 1.0, then the /// result will be `None`. /// - /// While `q` can be either 0.0 or 1.0, callers should prefer to use [`Summary::min`] and - /// [`Summary::max`] as the values will be the true values, and not an estimation. + /// If the 0.0 or 1.0 quantile is requested, this function will return self.min() or self.max() + /// instead of the estimated value. pub fn quantile(&self, q: f64) -> Option { if !(0.0..=1.0).contains(&q) || self.count() == 0 { return None; } - let ncount = self.negative.count(); - let pcount = self.positive.count(); - let zcount = self.zeroes; - let total = ncount + pcount + zcount; - let rank = (q * (total - 1) as f64) as usize; - - if rank < ncount { - // Quantile lands in the negative side. - let nq = 1.0 - (rank as f64 / ncount as f64); - self.negative.quantile(nq).expect("quantile should be valid at this point").map(|v| -v) - } else if rank >= ncount && rank < (ncount + zcount) { - // Quantile lands in the zero band. - Some(0.0) - } else { - // Quantile lands in the positive side. - let pq = (rank - (ncount + zcount)) as f64 / pcount as f64; - self.positive.quantile(pq).expect("quantile should be valid at this point") - } + self.sketch.quantile(q).expect("quantile should be valid at this point") + } + + /// Merge another Summary into this one. + /// + /// # Errors + /// + /// This function will return an error if the other Summary was not created with the same + /// parameters. + pub fn merge(&mut self, other: &Summary) -> Result<(), MergeError> { + self.sketch.merge(&other.sketch).map_err(|_| MergeError {})?; + Ok(()) } /// Gets the minimum value this summary has seen so far. pub fn min(&self) -> f64 { - self.min + self.sketch.min().unwrap_or(f64::INFINITY) } /// Gets the maximum value this summary has seen so far. pub fn max(&self) -> f64 { - self.max + self.sketch.max().unwrap_or(f64::NEG_INFINITY) } /// Whether or not this summary is empty. @@ -166,12 +135,7 @@ impl Summary { /// Gets the number of samples in this summary. pub fn count(&self) -> usize { - self.negative.count() + self.positive.count() + self.zeroes - } - - /// Gets the number of samples in this summary by zeroes, negative, and positive counts. - pub fn detailed_count(&self) -> (usize, usize, usize) { - (self.zeroes, self.negative.count(), self.positive.count()) + self.sketch.count() } /// Gets the estimized size of this summary, in bytes. @@ -179,10 +143,21 @@ impl Summary { /// In practice, this value should be very close to the actual size, but will not be entirely /// precise. pub fn estimated_size(&self) -> usize { - std::mem::size_of::() + ((self.positive.length() + self.negative.length()) * 8) + std::mem::size_of::() + (self.sketch.length() * 8) + } +} + +#[derive(Copy, Clone, Debug)] +pub struct MergeError {} + +impl fmt::Display for MergeError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "merge error") } } +impl std::error::Error for MergeError {} + #[cfg(test)] mod tests { use super::Summary; @@ -202,7 +177,10 @@ mod tests { #[test] fn test_basics() { - let mut summary = Summary::with_defaults(); + let alpha = 0.0001; + let max_bins = 32_768; + let min_value = 1.0e-9; + let mut summary = Summary::new(alpha, max_bins, min_value); assert!(summary.is_empty()); // Stretch the legs with a single value. @@ -210,23 +188,51 @@ mod tests { assert_eq!(summary.count(), 1); assert_relative_eq!(summary.min(), -420.42); assert_relative_eq!(summary.max(), -420.42); - assert_abs_diff_eq!(summary.quantile(0.1).expect("value should exist"), -420.42); - assert_abs_diff_eq!(summary.quantile(0.5).expect("value should exist"), -420.42); - assert_abs_diff_eq!(summary.quantile(0.99).expect("value should exist"), -420.42); + + let test_cases = vec![(0.1, -420.42), (0.5, -420.42), (0.9, -420.42)]; + for (q, val) in test_cases { + assert_relative_eq!( + summary.quantile(q).expect("value should exist"), + val, + max_relative = alpha + ); + } summary.add(420.42); + assert_eq!(summary.count(), 2); assert_relative_eq!(summary.min(), -420.42); assert_relative_eq!(summary.max(), 420.42); - assert_abs_diff_eq!(summary.quantile(0.49).expect("value should exist"), -420.42); + assert_relative_eq!( + summary.quantile(0.5).expect("value should exist"), + -420.42, + max_relative = alpha + ); + assert_relative_eq!( + summary.quantile(0.51).expect("value should exist"), + -420.42, + max_relative = alpha + ); summary.add(42.42); assert_eq!(summary.count(), 3); assert_relative_eq!(summary.min(), -420.42); assert_relative_eq!(summary.max(), 420.42); - assert_abs_diff_eq!(summary.quantile(0.4999999999).expect("value should exist"), -420.42); - assert_abs_diff_eq!(summary.quantile(0.5).expect("value should exist"), 42.42); - assert_abs_diff_eq!(summary.quantile(0.9999999999).expect("value should exist"), 42.42); + + let test_cases = vec![ + (0.333333, -420.42), + (0.333334, -420.42), + (0.666666, 42.42), + (0.666667, 42.42), + (0.999999, 42.42), + ]; + for (q, val) in test_cases { + assert_relative_eq!( + summary.quantile(q).expect("value should exist"), + val, + max_relative = alpha + ); + } } #[test] diff --git a/metrics-util/src/test_util.rs b/metrics-util/src/test_util.rs index 8d17afbc..cae3fc1d 100644 --- a/metrics-util/src/test_util.rs +++ b/metrics-util/src/test_util.rs @@ -1,18 +1,18 @@ -use metrics::{Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; use mockall::{ mock, - predicate::{self, eq}, + predicate::{self, always, eq}, Predicate, }; #[derive(Clone)] pub enum RecorderOperation { - DescribeCounter(KeyName, Option, &'static str), - DescribeGauge(KeyName, Option, &'static str), - DescribeHistogram(KeyName, Option, &'static str), - RegisterCounter(Key, Counter), - RegisterGauge(Key, Gauge), - RegisterHistogram(Key, Histogram), + DescribeCounter(KeyName, Option, SharedString), + DescribeGauge(KeyName, Option, SharedString), + DescribeHistogram(KeyName, Option, SharedString), + RegisterCounter(Key, Counter, &'static Metadata<'static>), + RegisterGauge(Key, Gauge, &'static Metadata<'static>), + RegisterHistogram(Key, Histogram, &'static Metadata<'static>), } impl RecorderOperation { @@ -27,11 +27,13 @@ impl RecorderOperation { RecorderOperation::DescribeHistogram(key_name, unit, desc) => { expect_describe_histogram(mock, key_name, unit, desc) } - RecorderOperation::RegisterCounter(key, counter) => { + RecorderOperation::RegisterCounter(key, counter, _) => { expect_register_counter(mock, key, counter) } - RecorderOperation::RegisterGauge(key, gauge) => expect_register_gauge(mock, key, gauge), - RecorderOperation::RegisterHistogram(key, histogram) => { + RecorderOperation::RegisterGauge(key, gauge, _) => { + expect_register_gauge(mock, key, gauge) + } + RecorderOperation::RegisterHistogram(key, histogram, _) => { expect_register_histogram(mock, key, histogram) } } @@ -51,14 +53,14 @@ impl RecorderOperation { RecorderOperation::DescribeHistogram(key_name, unit, desc) => { recorder.describe_histogram(key_name, unit, desc); } - RecorderOperation::RegisterCounter(key, _) => { - let _ = recorder.register_counter(&key); + RecorderOperation::RegisterCounter(key, _, metadata) => { + let _ = recorder.register_counter(&key, metadata); } - RecorderOperation::RegisterGauge(key, _) => { - let _ = recorder.register_gauge(&key); + RecorderOperation::RegisterGauge(key, _, metadata) => { + let _ = recorder.register_gauge(&key, metadata); } - RecorderOperation::RegisterHistogram(key, _) => { - let _ = recorder.register_histogram(&key); + RecorderOperation::RegisterHistogram(key, _, metadata) => { + let _ = recorder.register_histogram(&key, metadata); } } } @@ -68,12 +70,12 @@ mock! { pub BasicRecorder {} impl Recorder for BasicRecorder { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str); - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str); - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str); - fn register_counter(&self, key: &Key) -> Counter; - fn register_gauge(&self, key: &Key) -> Gauge; - fn register_histogram(&self, key: &Key) -> Histogram; + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString); + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString); + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString); + fn register_counter<'a>(&'a self, key: &'a Key, metadata: &'a Metadata<'a>) -> Counter; + fn register_gauge<'a>(&'a self, key: &'a Key, metadata: &'a Metadata<'a>) -> Gauge; + fn register_histogram<'a>(&'a self, key: &'a Key, metadata: &'a Metadata<'a>) -> Histogram; } } @@ -94,7 +96,7 @@ pub fn expect_describe_counter( mock: &mut MockBasicRecorder, key_name: KeyName, unit: Option, - description: &'static str, + description: SharedString, ) { mock.expect_describe_counter() .times(1) @@ -106,7 +108,7 @@ pub fn expect_describe_gauge( mock: &mut MockBasicRecorder, key_name: KeyName, unit: Option, - description: &'static str, + description: SharedString, ) { mock.expect_describe_gauge() .times(1) @@ -118,7 +120,7 @@ pub fn expect_describe_histogram( mock: &mut MockBasicRecorder, key_name: KeyName, unit: Option, - description: &'static str, + description: SharedString, ) { mock.expect_describe_histogram() .times(1) @@ -127,15 +129,15 @@ pub fn expect_describe_histogram( } pub fn expect_register_counter(mock: &mut MockBasicRecorder, key: Key, counter: Counter) { - mock.expect_register_counter().times(1).with(ref_eq(key)).return_const(counter); + mock.expect_register_counter().times(1).with(ref_eq(key), always()).return_const(counter); } pub fn expect_register_gauge(mock: &mut MockBasicRecorder, key: Key, gauge: Gauge) { - mock.expect_register_gauge().times(1).with(ref_eq(key)).return_const(gauge); + mock.expect_register_gauge().times(1).with(ref_eq(key), always()).return_const(gauge); } pub fn expect_register_histogram(mock: &mut MockBasicRecorder, key: Key, histogram: Histogram) { - mock.expect_register_histogram().times(1).with(ref_eq(key)).return_const(histogram); + mock.expect_register_histogram().times(1).with(ref_eq(key), always()).return_const(histogram); } fn ref_eq(value: T) -> impl Predicate { diff --git a/metrics/CHANGELOG.md b/metrics/CHANGELOG.md index 17cb3838..bc47838a 100644 --- a/metrics/CHANGELOG.md +++ b/metrics/CHANGELOG.md @@ -8,6 +8,81 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - ReleaseDate +### Added + +- Support for using `Arc` with `Cow<'a, T>`. + ([#402](https://github.com/metrics-rs/metrics/pull/402)) + + This will primarily allow using `Arc` for metric names and labels, where previously only + `&'static str` or `String` were allowed. There's still work to be done to also support labels in + this regard. + +### Changed + +- Make `Unit` methods return `&'static str` (instead of `&str`) where possible. ([#392](https://github.com/metrics-rs/metrics/pull/393)) + +## [0.21.1] - 2023-07-02 + +### Added + +- Added a `From` implementation for `KeyName` from `Cow<'static, str>`. + ([#378](https://github.com/metrics-rs/metrics/pull/378)) + +### Removed + +- Removed `metrics::set_recorder_racy` as it was intended to be used in `no_std` use cases, but + `metrics` is not currently compatible in `no_std` environments, so keeping `set_recorder_racy` + around was just API baggage. + +## [0.21.0] - 2023-04-16 + +### Added + +- A new module, `atomics`, exposes the atomic integer type that `CounterFn` and `GaugeFn` are + implemented for. This will publicly re-export the type for usage by downstream crates. (Credit to + [@repi](https://github.com/repi) for the original PR (#347) that did this.) + +### Changed + +- Bump MSRV to 1.61.0. +- `portable-atomic` is only used on 32-bit architectures. + +### Removed + +- Removed the `std-atomics` feature flag. + +## [0.20.1] - 2022-07-22 + +### Changed + +- Bumped the dependency on `metrics-macros` to correctly use the updated versions that are necessary + for handling the recent `&'static str` -> `SharedString` change to `Recorder::describe_*`. + + We'll also yank 0.20.0 once this is released to avoid the patch version triggering a breaking + change jump in transitive dependencies, and so people can't pick up a version of `metrics` that + doesn't actually work as it should. + +## [0.20.0] - 2022-07-20 + +### Changed + +- Changed `Recorder::describe_*` to take `SharedString` instead of `&'static str` for descriptions. (#312) +- Implemented `CounterFn` and `GaugeFn` for `portable_atomic::AtomicU64` (#313) +- Moved implementations of `CounterFn` and `GaugeFn` for `std::sync::atomic::AtomicU64` behind a + default feature flag. + +## [0.19.0] - 2022-05-30 + +### Fixed + +- Small typo in the documentation. ([#286](https://github.com/metrics-rs/metrics/pull/286)) + +### Changed + +- Refactored the global recorder instance, namely around how it gets set and documenting the safety guarantees of + methods related to setting and unsetting it. ([#302](https://github.com/metrics-rs/metrics/pull/302)) +- Fixed an issue with pointer provenance in `metrics::Cow`. ([#303](https://github.com/metrics-rs/metrics/pull/303)) + ## [0.18.1] - 2022-03-10 ### Added diff --git a/metrics/Cargo.toml b/metrics/Cargo.toml index 08f2c631..e16cabc3 100644 --- a/metrics/Cargo.toml +++ b/metrics/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "metrics" -version = "0.18.1" +version = "0.21.1" authors = ["Toby Lawrence "] edition = "2018" +rust-version = "1.61.0" license = "MIT" @@ -15,8 +16,6 @@ readme = "README.md" categories = ["development-tools::debugging"] keywords = ["metrics", "facade"] -build = "build.rs" - [lib] bench = false @@ -25,11 +24,15 @@ name = "macros" harness = false [dependencies] -metrics-macros = { version = "^0.5", path = "../metrics-macros" } -ahash = { version = "0.7", default-features = false } +ahash = { version = "0.8", default-features = false } + +[target.'cfg(target_pointer_width = "32")'.dependencies] +portable-atomic = { version = "1", default-features = false, features = [ + "fallback", +] } [dev-dependencies] log = "0.4" -criterion = { version = "0.3", default-features = false, features = ["html_reports", "cargo_bench_support"] } +criterion = { version = "=0.3.3", default-features = false } rand = "0.8" trybuild = "1" diff --git a/metrics/RELEASES.md b/metrics/RELEASES.md index 54f81648..cdf7bc12 100644 --- a/metrics/RELEASES.md +++ b/metrics/RELEASES.md @@ -1,4 +1,5 @@ # Releases + Unlike the [CHANGELOG](CHANGELOG.md), this file tracks more complicated changes that required long-form description and would be too verbose for the changelog alone. @@ -6,6 +7,145 @@ long-form description and would be too verbose for the changelog alone. ## [Unreleased] - ReleaseDate +### Metric metadata + +Metrics now support a limited set of metadata field, which can be added to provide for context about +the metric in terms of where it originates from as well as its verbosity. + +In the grand vision of the `metrics` crate, where everyone uses the crate to emit metrics and then +users get those metrics in their application for free... the biggest problem is that users had no +way to actually filter out metrics they didn't want. Metric metadata aims to provide a solution to +this problem. + +A new type `Metadata<'a>` has been added to track all of this information, which includes **module +path**, a **target**, and a **level**. These fields map almost directly to +[`tracing`](https://docs.rs/tracing) -- the inspiration for adding this metadata support -- and +provide the ability to: + +- group/filter metrics by where they're defined (module path) +- group/filter metrics by where they're emitted from (target) +- group/filter metrics by their verbosity (level) + +`Metadata<'a>` is passed into the `Recorder` API when registering a metric so that exporters can +capture it and utilize it. + +#### Examples + +As an example, users may wish to filter out metrics defined by a particular crate because they don't +care about them at all. While they might have previously been able to get lucky and simply filter +the metrics by a common prefix, this still allows for changes to the metric names to breaking the +filter configuration. If we could instead filter by module path, where we can simply use the crate +name itself, then we'd catch all metrics for that crate regardless of their name and regardless of +the crate version. + +Similarly, as another example, users may wish to only emit common metrics related to operation of +their application/service in order to consume less resources, pay less money for the ingest/storage +of the metrics, and so on. During an outage, or when debugging an issue, though, they may wish to +increase the verbosity of metrics they emit in order to capture more granular detail. Being able to +filter by level now provides a mechanism to do so. + +#### Usage + +First, it needs to be said that nothing in the core `metrics` crates actually utilizes this +metadata yet. We'll add support in the future to existing layers, such as the +[filter][filter_layer_docs] layer, in order to take advantage of this support. + +With that said, actually setting this metadata is very easy! As a refresher, you'd normally emit +metrics with something like this: + +```rust +metrics::increment_counter!("my_counter"); +``` + +Now, you can specify the additional metadata attributes as fields at the beginning of the macro +call. This applies to all of the "emission" macros for counters, gauges, and histograms: + +```rust +metrics::increment_counter!(target: "myapp", "my_counter"); + +metrics::increment_gauge!(level: metrics::Level::DEBUG, "my_gauge", 42.2); + +metrics::histogram!(target: "myapp", level: metrics::Level::DEBUG, "my_histogram", 180.1); +``` + +These metrics will have the relevant metadata field set, and all of them will get the module path +provided automatically, as well. + +### Macros overhaul + +In this release, we've reworked the macros to both simplify their implementation and to hopefully +provide a more ergonomic experience for users. + +At a high level, we've: + +- removed all the various macros that were tied to specific _operations_ (e.g. `increment_counter!` + for incrementing a counter) and replaced them with one macro per metric type +- removed the standalone registration macros (e.g. `register_counter!`) +- exposed the operations as methods on the metric handles themselves +- switched from using procedural macros to declarative macros + +#### Fewer macros, more ergonomic usage + +Users no longer need to remember the specific macro to use for a given metric operation, such as +`increment_gauge!` or `decrement_gauge!`. Instead, if the user knows they're working with a gauge, +they simply call `gauge!(...)` to get the handle, and chain on a method call to perform the +operation, such as `gauge!(...).increment(42.2)`. + +Additionally, because we've condensed the registration macros into the new, simplified macros, the +same macro is used whether registering the metric to get a handle, or simply performing an operation +on the metric all at once. + +Let's look at a few examples: + +```rust +// This is the old style of registering a metric and then performing an operation on it. +// +// We're working with a counter here. +let counter_handle = metrics::register_counter!("my_counter"); +counter_handle.increment(1); + +metrics::increment_counter!("my_counter"); + +// Now let's use the new, simplified macros instead: +let counter_handle = metrics::counter!("my_counter"); +counter_handle.increment(1); + +metrics::counter!("my_counter").increment(1); +``` + +As you can see, users no longer need to know about as many macros, and their usage is consistent +whether working with a metric handle that is held long-term, or chaining the method call inline with +the macro call. As a benefit, this also means that IDE completion will be better in some situations, +as support for autocompletion of method calls is generally well supported, while macro +autocompletion is effectively nonexistent. + +#### Declarative macros + +As part of this rework, the macros have also been rewritten declaratively. While the macro code +itself is slightly more complicated/verbose, it has a major benefit that the `metrics-macros` crate +was able to be removed. This is one less dependency that has to be compiled, which should hopefully +help with build times, even if only slightly. + +[filter_layer_docs]: https://docs.rs/metrics-util/latest/metrics_util/layers/struct.FilterLayer.html + +## [0.21.1] - 2023-07-02 + +- No notable changes. + +## [0.21.0] - 2023-04-16 + +- No notable changes. + +## [0.20.1] - 2022-07-22 + +- No notable changes. + +## [0.20.0] - 2022-07-20 + +- No notable changes. + +## [0.19.0] - 2022-05-30 + - No notable changes. ## [0.18.1] - 2022-03-10 @@ -15,6 +155,7 @@ long-form description and would be too verbose for the changelog alone. ## [0.18.0] - 2022-01-14 ### Switch to metric handles + `metrics` has now switched to "metric handles." This requires a small amount of backstory. #### Evolution of data storage in `metrics` diff --git a/metrics/benches/macros.rs b/metrics/benches/macros.rs index 721eead0..7ece0dcc 100644 --- a/metrics/benches/macros.rs +++ b/metrics/benches/macros.rs @@ -3,67 +3,82 @@ extern crate criterion; use criterion::Criterion; -use metrics::{counter, Counter, Gauge, Histogram, Key, KeyName, Recorder, Unit}; +use metrics::{ + counter, Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit, +}; use rand::{thread_rng, Rng}; #[derive(Default)] struct TestRecorder; impl Recorder for TestRecorder { - fn describe_counter(&self, _: KeyName, _: Option, _: &'static str) {} - fn describe_gauge(&self, _: KeyName, _: Option, _: &'static str) {} - fn describe_histogram(&self, _: KeyName, _: Option, _: &'static str) {} - fn register_counter(&self, _: &Key) -> Counter { + fn describe_counter(&self, _: KeyName, _: Option, _: SharedString) {} + fn describe_gauge(&self, _: KeyName, _: Option, _: SharedString) {} + fn describe_histogram(&self, _: KeyName, _: Option, _: SharedString) {} + fn register_counter(&self, _: &Key, _: &Metadata<'_>) -> Counter { Counter::noop() } - fn register_gauge(&self, _: &Key) -> Gauge { + fn register_gauge(&self, _: &Key, _: &Metadata<'_>) -> Gauge { Gauge::noop() } - fn register_histogram(&self, _: &Key) -> Histogram { + fn register_histogram(&self, _: &Key, _: &Metadata<'_>) -> Histogram { Histogram::noop() } } fn reset_recorder() { - let recorder = unsafe { &*Box::into_raw(Box::new(TestRecorder::default())) }; - unsafe { metrics::set_recorder_racy(recorder).unwrap() } + unsafe { + metrics::clear_recorder(); + } + metrics::set_boxed_recorder(Box::new(TestRecorder::default())).unwrap() } fn macro_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("macros"); group.bench_function("uninitialized/no_labels", |b| { - metrics::clear_recorder(); + unsafe { + metrics::clear_recorder(); + } b.iter(|| { - counter!("counter_bench", 42); + counter!("counter_bench").increment(42); }) }); group.bench_function("uninitialized/with_static_labels", |b| { - metrics::clear_recorder(); + unsafe { + metrics::clear_recorder(); + } b.iter(|| { - counter!("counter_bench", 42, "request" => "http", "svc" => "admin"); + counter!("counter_bench", "request" => "http", "svc" => "admin").increment(42); }) }); group.bench_function("initialized/no_labels", |b| { reset_recorder(); b.iter(|| { - counter!("counter_bench", 42); + counter!("counter_bench").increment(42); }); - metrics::clear_recorder(); + unsafe { + metrics::clear_recorder(); + } }); group.bench_function("initialized/with_static_labels", |b| { reset_recorder(); b.iter(|| { - counter!("counter_bench", 42, "request" => "http", "svc" => "admin"); + counter!("counter_bench", "request" => "http", "svc" => "admin").increment(42); }); - metrics::clear_recorder(); + unsafe { + metrics::clear_recorder(); + } }); group.bench_function("initialized/with_dynamic_labels", |b| { let label_val = thread_rng().gen::().to_string(); reset_recorder(); b.iter(move || { - counter!("counter_bench", 42, "request" => "http", "uid" => label_val.clone()); + counter!("counter_bench", "request" => "http", "uid" => label_val.clone()) + .increment(42); }); - metrics::clear_recorder(); + unsafe { + metrics::clear_recorder(); + } }); group.finish(); } diff --git a/metrics/build.rs b/metrics/build.rs deleted file mode 100644 index b5c13da8..00000000 --- a/metrics/build.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! This build script detects target platforms that lack proper support for -//! atomics and sets `cfg` flags accordingly. -use std::env; - -fn main() { - // CAS is not available on thumbv6. - let target = env::var("TARGET").unwrap(); - if !target.starts_with("thumbv6") { - println!("cargo:rustc-cfg=atomic_cas"); - } - - println!("cargo:rerun-if-changed=build.rs"); -} diff --git a/metrics/examples/basic.rs b/metrics/examples/basic.rs index c678bf0d..9b2da7e4 100644 --- a/metrics/examples/basic.rs +++ b/metrics/examples/basic.rs @@ -8,9 +8,8 @@ use std::sync::Arc; use metrics::{ - absolute_counter, counter, decrement_gauge, describe_counter, describe_gauge, - describe_histogram, gauge, histogram, increment_counter, increment_gauge, register_counter, - register_gauge, register_histogram, KeyName, + counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram, KeyName, + Metadata, SharedString, }; use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, Recorder, Unit}; @@ -50,7 +49,7 @@ impl HistogramFn for PrintHandle { struct PrintRecorder; impl Recorder for PrintRecorder { - fn describe_counter(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_counter(&self, key_name: KeyName, unit: Option, description: SharedString) { println!( "(counter) registered key {} with unit {:?} and description {:?}", key_name.as_str(), @@ -59,7 +58,7 @@ impl Recorder for PrintRecorder { ); } - fn describe_gauge(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_gauge(&self, key_name: KeyName, unit: Option, description: SharedString) { println!( "(gauge) registered key {} with unit {:?} and description {:?}", key_name.as_str(), @@ -68,7 +67,7 @@ impl Recorder for PrintRecorder { ); } - fn describe_histogram(&self, key_name: KeyName, unit: Option, description: &'static str) { + fn describe_histogram(&self, key_name: KeyName, unit: Option, description: SharedString) { println!( "(histogram) registered key {} with unit {:?} and description {:?}", key_name.as_str(), @@ -77,15 +76,15 @@ impl Recorder for PrintRecorder { ); } - fn register_counter(&self, key: &Key) -> Counter { + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { Counter::from_arc(Arc::new(PrintHandle(key.clone()))) } - fn register_gauge(&self, key: &Key) -> Gauge { + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { Gauge::from_arc(Arc::new(PrintHandle(key.clone()))) } - fn register_histogram(&self, key: &Key) -> Histogram { + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { Histogram::from_arc(Arc::new(PrintHandle(key.clone()))) } } @@ -119,54 +118,59 @@ fn main() { ); // And registering them: - let counter1 = register_counter!("test_counter"); + let counter1 = counter!("test_counter"); counter1.increment(1); - let counter2 = register_counter!("test_counter", "type" => "absolute"); + let counter2 = counter!("test_counter", "type" => "absolute"); counter2.absolute(42); - let gauge1 = register_gauge!("test_gauge"); + let gauge1 = gauge!("test_gauge"); gauge1.increment(1.0); - let gauge2 = register_gauge!("test_gauge", "type" => "decrement"); + let gauge2 = gauge!("test_gauge", "type" => "decrement"); gauge2.decrement(1.0); - let gauge3 = register_gauge!("test_gauge", "type" => "set"); + let gauge3 = gauge!("test_gauge", "type" => "set"); gauge3.set(3.1459); - let histogram1 = register_histogram!("test_histogram"); + let histogram1 = histogram!("test_histogram"); histogram1.record(0.57721); // All the supported permutations of `counter!` and its increment/absolute versions: - counter!("bytes_sent", 64); - counter!("bytes_sent", 64, "listener" => "frontend"); - counter!("bytes_sent", 64, "listener" => "frontend", "server" => server_name.clone()); - counter!("bytes_sent", 64, common_labels); - - increment_counter!("requests_processed"); - increment_counter!("requests_processed", "request_type" => "admin"); - increment_counter!("requests_processed", "request_type" => "admin", "server" => server_name.clone()); - increment_counter!("requests_processed", common_labels); - - absolute_counter!("bytes_sent", 64); - absolute_counter!("bytes_sent", 64, "listener" => "frontend"); - absolute_counter!("bytes_sent", 64, "listener" => "frontend", "server" => server_name.clone()); - absolute_counter!("bytes_sent", 64, common_labels); + counter!("bytes_sent").increment(64); + counter!("bytes_sent", "listener" => "frontend").increment(64); + counter!("bytes_sent", "listener" => "frontend", "server" => server_name.clone()).increment(64); + counter!("bytes_sent", common_labels).increment(64); + + counter!("requests_processed").increment(1); + counter!("requests_processed", "request_type" => "admin").increment(1); + counter!("requests_processed", "request_type" => "admin", "server" => server_name.clone()) + .increment(1); + counter!("requests_processed", common_labels).increment(1); + + counter!("bytes_sent").absolute(64); + counter!("bytes_sent", "listener" => "frontend").absolute(64); + counter!("bytes_sent", "listener" => "frontend", "server" => server_name.clone()).absolute(64); + counter!("bytes_sent", common_labels).absolute(64); // All the supported permutations of `gauge!` and its increment/decrement versions: - gauge!("connection_count", 300.0); - gauge!("connection_count", 300.0, "listener" => "frontend"); - gauge!("connection_count", 300.0, "listener" => "frontend", "server" => server_name.clone()); - gauge!("connection_count", 300.0, common_labels); - increment_gauge!("connection_count", 300.0); - increment_gauge!("connection_count", 300.0, "listener" => "frontend"); - increment_gauge!("connection_count", 300.0, "listener" => "frontend", "server" => server_name.clone()); - increment_gauge!("connection_count", 300.0, common_labels); - decrement_gauge!("connection_count", 300.0); - decrement_gauge!("connection_count", 300.0, "listener" => "frontend"); - decrement_gauge!("connection_count", 300.0, "listener" => "frontend", "server" => server_name.clone()); - decrement_gauge!("connection_count", 300.0, common_labels); + gauge!("connection_count").set(300.0); + gauge!("connection_count", "listener" => "frontend").set(300.0); + gauge!("connection_count", "listener" => "frontend", "server" => server_name.clone()) + .set(300.0); + gauge!("connection_count", common_labels).set(300.0); + gauge!("connection_count").increment(300.0); + gauge!("connection_count", "listener" => "frontend").increment(300.0); + gauge!("connection_count", "listener" => "frontend", "server" => server_name.clone()) + .increment(300.0); + gauge!("connection_count", common_labels).increment(300.0); + gauge!("connection_count").decrement(300.0); + gauge!("connection_count", "listener" => "frontend").decrement(300.0); + gauge!("connection_count", "listener" => "frontend", "server" => server_name.clone()) + .decrement(300.0); + gauge!("connection_count", common_labels).decrement(300.0); // All the supported permutations of `histogram!`: - histogram!("svc.execution_time", 70.0); - histogram!("svc.execution_time", 70.0, "type" => "users"); - histogram!("svc.execution_time", 70.0, "type" => "users", "server" => server_name.clone()); - histogram!("svc.execution_time", 70.0, common_labels); + histogram!("svc.execution_time").record(70.0); + histogram!("svc.execution_time", "type" => "users").record(70.0); + histogram!("svc.execution_time", "type" => "users", "server" => server_name.clone()) + .record(70.0); + histogram!("svc.execution_time", common_labels).record(70.0); } diff --git a/metrics/src/atomics.rs b/metrics/src/atomics.rs new file mode 100644 index 00000000..15681143 --- /dev/null +++ b/metrics/src/atomics.rs @@ -0,0 +1,64 @@ +//! Atomic types used for metrics. +//! +//! As the most commonly used types for metrics storage are atomic integers, implementations of +//! `CounterFn` and `GaugeFn` must be provided in this crate due to Rust's "orphan rules", which +//! disallow a crate from implementing a foreign trait on a foreign type. +//! +//! Further, we always require an atomic integer of a certain size regardless of whether the +//! standard library exposes an atomic integer of that size for the target architecture. +//! +//! As such, the atomic types that we provide handle implementations for are publicly re-exporter +//! here for downstream crates to utilize. + +use std::sync::atomic::Ordering; + +#[cfg(target_pointer_width = "32")] +pub use portable_atomic::AtomicU64; +#[cfg(not(target_pointer_width = "32"))] +pub use std::sync::atomic::AtomicU64; + +use super::{CounterFn, GaugeFn}; + +impl CounterFn for AtomicU64 { + fn increment(&self, value: u64) { + let _ = self.fetch_add(value, Ordering::Release); + } + + fn absolute(&self, value: u64) { + let _ = self.fetch_max(value, Ordering::AcqRel); + } +} + +impl GaugeFn for AtomicU64 { + fn increment(&self, value: f64) { + loop { + let result = self.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |curr| { + let input = f64::from_bits(curr); + let output = input + value; + Some(output.to_bits()) + }); + + if result.is_ok() { + break; + } + } + } + + fn decrement(&self, value: f64) { + loop { + let result = self.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |curr| { + let input = f64::from_bits(curr); + let output = input - value; + Some(output.to_bits()) + }); + + if result.is_ok() { + break; + } + } + } + + fn set(&self, value: f64) { + let _ = self.swap(value.to_bits(), Ordering::AcqRel); + } +} diff --git a/metrics/src/common.rs b/metrics/src/common.rs index c92fedf9..6cabedc0 100644 --- a/metrics/src/common.rs +++ b/metrics/src/common.rs @@ -6,12 +6,15 @@ use crate::cow::Cow; /// An allocation-optimized string. /// -/// We specify `SharedString` to attempt to get the best of both worlds: flexibility to provide a -/// static or dynamic (owned) string, while retaining the performance benefits of being able to -/// take ownership of owned strings and borrows of completely static strings. +/// `SharedString` uses a custom copy-on-write implementation that is optimized for metric keys, +/// providing ergonomic sharing of single instances, or slices, of strings and labels. This +/// copy-on-write implementation is optimized to allow for constant-time construction (using static +/// values), as well as accepting owned values and values shared through [`Arc`](std::sync::Arc). /// -/// `SharedString` can be converted to from either `&'static str` or `String`, with a method, -/// `const_str`, from constructing `SharedString` from `&'static str` in a `const` fashion. +/// End users generally will not need to interact with this type directly, as the top-level macros +/// (`counter!`, etc), as well as the various conversion implementations +/// ([`From`](std::convert::From)), generally allow users to pass whichever variant of a value +/// (static, owned, shared) is best for them. pub type SharedString = Cow<'static, str>; /// Key-specific hashing algorithm. @@ -121,7 +124,7 @@ pub enum Unit { impl Unit { /// Gets the string form of this `Unit`. - pub fn as_str(&self) -> &str { + pub fn as_str(&self) -> &'static str { match self { Unit::Count => "count", Unit::Percent => "percent", @@ -149,7 +152,7 @@ impl Unit { /// it would be `ns`. /// /// Not all units have a meaningful display label and so some may be empty. - pub fn as_canonical_label(&self) -> &str { + pub fn as_canonical_label(&self) -> &'static str { match self { Unit::Count => "", Unit::Percent => "%", diff --git a/metrics/src/cow.rs b/metrics/src/cow.rs index 5ff200b5..7087fcb5 100644 --- a/metrics/src/cow.rs +++ b/metrics/src/cow.rs @@ -1,35 +1,182 @@ -use crate::label::Label; -use alloc::borrow::Borrow; -use alloc::string::String; -use alloc::vec::Vec; -use core::cmp::Ordering; -use core::fmt; -use core::hash::{Hash, Hasher}; -use core::marker::PhantomData; -use core::mem::ManuallyDrop; -use core::ptr::{slice_from_raw_parts, NonNull}; - -/// A clone-on-write smart pointer with an optimized memory layout. +use std::{ + borrow::Borrow, + cmp::Ordering, + fmt, + hash::{Hash, Hasher}, + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr::{slice_from_raw_parts, NonNull}, + sync::Arc, +}; + +#[derive(Clone, Copy)] +enum Kind { + Owned, + Borrowed, + Shared, +} + +/// A clone-on-write smart pointer with an optimized memory layout, based on `beef`. +/// +/// # Strings, strings everywhere +/// +/// In `metrics`, strings are arguably the most common data type used despite the fact that metrics +/// are measuring numerical values. Both the name of a metric, and its labels, are strings: emitting +/// a metric may involve one string, or 10 strings. Many of these strings tend to be used over and +/// over during the life of the process, as well. +/// +/// In order to achieve and maintain a high level of performance, we use a "clone-on-write" smart +/// pointer to handle passing these strings around. Doing so allows us to potentially avoid having +/// to allocate entire copies of a string, instead using a lightweight smart pointer that can live +/// on the stack. +/// +/// # Why not `std::borrow::Cow`? +/// +/// The standard library already provides a clone-on-write smart pointer, `std::borrow::Cow`, which +/// works well in many cases. However, `metrics` strives to provide minimal overhead where possible, +/// and so `std::borrow::Cow` falls down in one particular way: it uses an enum representation which +/// consumes an additional word of storage. +/// +/// As an example, let's look at strings. A string in `std::borrow::Cow` implies that `T` is `str`, +/// and the owned version of `str` is simply `String`. Thus, for `std::borrow::Cow`, the in-memory +/// layout looks like this: +/// +/// ```text +/// Padding +/// | +/// v +/// +--------------+-------------+--------------+--------------+ +/// stdlib Cow::Borrowed: | Enum Tag | Pointer | Length | XXXXXXXX | +/// +--------------+-------------+--------------+--------------+ +/// +--------------+-------------+--------------+--------------+ +/// stdlib Cow::Owned: | Enum Tag | Pointer | Length | Capacity | +/// +--------------+-------------+--------------+--------------+ +/// ``` +/// +/// As you can see, you pay a memory size penalty to be able to wrap an owned string. This +/// additional word adds minimal overhead, but we can easily avoid it with some clever logic around +/// the values of the length and capacity fields. +/// +/// There is an existing crate that does just that: `beef`. Instead of using an enum, it is simply a +/// struct that encodes the discriminant values in the length and capacity fields directly. If we're +/// wrapping a borrowed value, we can infer that the "capacity" will always be zero, as we only need +/// to track the capacity when we're wrapping an owned value, in order to be able to recreate the +/// underlying storage when consuming the smart pointer, or dropping it. Instead of the above +/// layout, `beef` looks like this: +/// +/// ```text +/// +-------------+--------------+----------------+ +/// `beef` Cow (borrowed): | Pointer | Length (N) | Capacity (0) | +/// +-------------+--------------+----------------+ +/// +-------------+--------------+----------------+ +/// `beef` Cow (owned): | Pointer | Length (N) | Capacity (M) | +/// +-------------+--------------+----------------+ +/// ``` +/// +/// # Why not `beef`? +/// +/// Up until this point, it might not be clear why we didn't just use `beef`. In truth, our design +/// is fundamentally based on `beef`. Crucially, however, `beef` did not/still does not support +/// `const` construction for generic slices. Remember how we mentioned labels? The labels of a +/// metric `are `[Label]` under-the-hood, and so without a way to construct them in a `const` +/// fashion, our previous work to allow entirely static keys would not be possible. +/// +/// Thus, we forked `beef` and copied into directly into `metrics` so that we could write a +/// specialized `const` constructor for `[Label]`. +/// +/// This is why we have our own `Cow` bundled into `metrics` directly, which is based on `beef`. In +/// doing so, we can experiment with more interesting optimizations, and, as mentioned above, we can +/// add const methods to support all of the types involved in statically building metrics keys. +/// +/// # What we do that `beef` doesn't do +/// +/// It was already enough to use our own implementation for the specialized `const` capabilities, +/// but we've taken things even further in a key way: support for `Arc`-wrapped values. +/// +/// ## `Arc`-wrapped values +/// +/// For many strings, there is still a desire to share them cheaply even when they are constructed +/// at run-time. Remember, cloning a `Cow` of an owned value means cloning the value itself, so we +/// need another level of indirection to allow the cheap sharing, which is where `Arc` can +/// provide value. +/// +/// Users can construct a `Arc`, where `T` is lined up with the `T` of `metrics::Cow`, and use +/// that as the initial value instead. When `Cow` is cloned, we end up cloning the underlying +/// `Arc` instead, avoiding a new allocation. `Arc` still handles all of the normal logic +/// necessary to know when the wrapped value must be dropped, and how many live references to the +/// value that there are, and so on. +/// +/// We handle this by relying on an invariant of `Vec`: it never allocates more than `isize::MAX` +/// [1]. This lets us derive the following truth table of the valid combinations of length/capacity: +/// +/// ```text +/// Length (N) Capacity (M) +/// +---------------+----------------+ +/// Borrowed (&T): | N | 0 | +/// +---------------+----------------+ +/// Owned (T::ToOwned): | N | M < usize::MAX | +/// +---------------+----------------+ +/// Shared (Arc): | N | usize::MAX | +/// +---------------+----------------+ +/// ``` +/// +/// As we only implement `Cow` for types where their owned variants are either explicitly or +/// implicitly backed by `Vec<_>`, we know that our capacity will never be `usize::MAX`, as it is +/// limited to `isize::MAX`, and thus we can safely encode our "shared" state within the capacity +/// field. +/// +/// # Notes +/// +/// [1] - technically, `Vec` can have a capacity greater than `isize::MAX` when storing +/// zero-sized types, but we don't do that here, so we always enforce that an owned version's +/// capacity cannot be `usize::MAX` when constructing `Cow`. pub struct Cow<'a, T: Cowable + ?Sized + 'a> { - /// Pointer to data. ptr: NonNull, - - /// Pointer metadata: length and capacity. - meta: Metadata, - - /// Lifetime marker. - marker: PhantomData<&'a T>, + metadata: Metadata, + _lifetime: PhantomData<&'a T>, } impl Cow<'_, T> where T: Cowable + ?Sized, { - #[inline] - pub fn owned(val: T::Owned) -> Self { - let (ptr, meta) = T::owned_into_parts(val); + fn from_parts(ptr: NonNull, metadata: Metadata) -> Self { + Self { ptr, metadata, _lifetime: PhantomData } + } + + /// Creates a pointer to an owned value, consuming it. + pub fn from_owned(owned: T::Owned) -> Self { + let (ptr, metadata) = T::owned_into_parts(owned); + + // This check is partially to guard against the semantics of `Vec` changing in the + // future, and partially to ensure that we don't somehow implement `Cowable` for a type + // where its owned version is backed by a vector of ZSTs, where the capacity could + // _legitimately_ be `usize::MAX`. + if metadata.capacity() == usize::MAX { + panic!("Invalid capacity of `usize::MAX` for owned value."); + } + + Self::from_parts(ptr, metadata) + } + + /// Creates a pointer to a shared value. + pub fn from_shared(arc: Arc) -> Self { + let (ptr, metadata) = T::shared_into_parts(arc); + Self::from_parts(ptr, metadata) + } + + /// Extracts the owned data. + /// + /// Clones the data if it is not already owned. + pub fn into_owned(self) -> ::Owned { + // We need to ensure that our own `Drop` impl does _not_ run because we're simply + // transferring ownership of the value back to the caller. For borrowed values, this is + // naturally a no-op because there's nothing to drop, but for owned values, like `String` or + // `Arc`, we wouldn't want to double drop. + let cow = ManuallyDrop::new(self); - Cow { ptr, meta, marker: PhantomData } + T::owned_from_parts(cow.ptr, &cow.metadata) } } @@ -37,71 +184,69 @@ impl<'a, T> Cow<'a, T> where T: Cowable + ?Sized, { - #[inline] - pub fn borrowed(val: &'a T) -> Self { - let (ptr, meta) = T::ref_into_parts(val); + /// Creates a pointer to a borrowed value. + pub fn from_borrowed(borrowed: &'a T) -> Self { + let (ptr, metadata) = T::borrowed_into_parts(borrowed); - Cow { ptr, meta, marker: PhantomData } + Self::from_parts(ptr, metadata) } +} - #[inline] - pub fn into_owned(self) -> T::Owned { - let cow = ManuallyDrop::new(self); - - if cow.is_borrowed() { - unsafe { T::clone_from_parts(cow.ptr, &cow.meta) } - } else { - unsafe { T::owned_from_parts(cow.ptr, &cow.meta) } - } - } +impl<'a, T> Cow<'a, [T]> +where + T: Clone, +{ + pub const fn const_slice(val: &'a [T]) -> Cow<'a, [T]> { + // SAFETY: We can never create a null pointer by casting a reference to a pointer. + let ptr = unsafe { NonNull::new_unchecked(val.as_ptr() as *mut _) }; + let metadata = Metadata::borrowed(val.len()); - #[inline] - pub fn is_borrowed(&self) -> bool { - self.meta.capacity() == 0 + Self { ptr, metadata, _lifetime: PhantomData } } +} - #[inline] - pub fn is_owned(&self) -> bool { - self.meta.capacity() != 0 - } +impl<'a> Cow<'a, str> { + pub const fn const_str(val: &'a str) -> Self { + // SAFETY: We can never create a null pointer by casting a reference to a pointer. + let ptr = unsafe { NonNull::new_unchecked(val.as_ptr() as *mut _) }; + let metadata = Metadata::borrowed(val.len()); - #[inline] - fn borrow(&self) -> &T { - unsafe { &*T::ref_from_parts(self.ptr, &self.meta) } + Self { ptr, metadata, _lifetime: PhantomData } } } -// Implementations of constant functions for creating `Cow` via static strings, static string -// slices, and static label slices. -impl<'a> Cow<'a, str> { - pub const fn const_str(val: &'a str) -> Self { - Cow { - // We are casting *const T to *mut T, however for all borrowed values - // this raw pointer is only ever dereferenced back to &T. - ptr: unsafe { NonNull::new_unchecked(val.as_ptr() as *mut u8) }, - meta: Metadata::from_ref(val.len()), - marker: PhantomData, - } +impl Deref for Cow<'_, T> +where + T: Cowable + ?Sized, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + let borrowed_ptr = T::borrowed_from_parts(self.ptr, &self.metadata); + + // SAFETY: We only ever hold a pointer to a borrowed value of at least the lifetime of + // `Self`, or an owned value which we have ownership of (albeit indirectly when using + // `Arc`), so our pointer is always valid and live for derefencing. + unsafe { borrowed_ptr.as_ref().unwrap() } } } -impl<'a> Cow<'a, [Cow<'static, str>]> { - pub const fn const_slice(val: &'a [Cow<'static, str>]) -> Self { - Cow { - ptr: unsafe { NonNull::new_unchecked(val.as_ptr() as *mut Cow<'static, str>) }, - meta: Metadata::from_ref(val.len()), - marker: PhantomData, - } +impl Clone for Cow<'_, T> +where + T: Cowable + ?Sized, +{ + fn clone(&self) -> Self { + let (ptr, metadata) = T::clone_from_parts(self.ptr, &self.metadata); + Self { ptr, metadata, _lifetime: PhantomData } } } -impl<'a> Cow<'a, [Label]> { - pub const fn const_slice(val: &'a [Label]) -> Self { - Cow { - ptr: unsafe { NonNull::new_unchecked(val.as_ptr() as *mut Label) }, - meta: Metadata::from_ref(val.len()), - marker: PhantomData, - } +impl Drop for Cow<'_, T> +where + T: Cowable + ?Sized, +{ + fn drop(&mut self) { + T::drop_from_parts(self.ptr, &self.metadata); } } @@ -111,7 +256,7 @@ where { #[inline] fn hash(&self, state: &mut H) { - self.borrow().hash(state) + self.deref().hash(state) } } @@ -122,7 +267,7 @@ where { #[inline] fn default() -> Self { - Cow::borrowed(Default::default()) + Cow::from_borrowed(Default::default()) } } @@ -135,7 +280,7 @@ where { #[inline] fn partial_cmp(&self, other: &Cow<'_, B>) -> Option { - PartialOrd::partial_cmp(self.borrow(), other.borrow()) + PartialOrd::partial_cmp(self.deref(), other.deref()) } } @@ -145,7 +290,7 @@ where { #[inline] fn cmp(&self, other: &Self) -> Ordering { - Ord::cmp(self.borrow(), other.borrow()) + Ord::cmp(self.deref(), other.deref()) } } @@ -155,77 +300,44 @@ where { #[inline] fn from(val: &'a T) -> Self { - Cow::borrowed(val) - } -} - -impl From> for Cow<'_, str> { - #[inline] - fn from(s: std::borrow::Cow<'static, str>) -> Self { - match s { - std::borrow::Cow::Borrowed(bs) => Cow::borrowed(bs), - std::borrow::Cow::Owned(os) => Cow::owned(os), - } - } -} - -impl From for Cow<'_, str> { - #[inline] - fn from(s: String) -> Self { - Cow::owned(s) - } -} - -impl From> for Cow<'_, [Label]> { - #[inline] - fn from(v: Vec