Health Metrics for data-pipeline + Dogstatsd-Client Crate (#638)

* move dogstatsd-client to separate crate so it can be shared * adding some initial stats and tests * make traceexporter mut * miri can ignore unit test * refactor dogstatsd-client to not need mutability * use vec default to avoid unneeded allocation * fmt * add dogstatsd-client to docker build tool * clean up cargo.toml * remove unneeded dep * use AsRef for metric strings, improve perf and simplify usage * simplify emitting stats in data-pipeline * fix lint * Remove copy of tags and pass to dogstatsd client by reference * Update licenses * Allow IntoIterator tags to support static and dynamic tags: WIP * Fix lifetimes of DogstatsDAction Signed-off-by: Bob Weinand <[email protected]> * Actually also allow &Vec<Tag> and &[Tag] Signed-off-by: Bob Weinand <[email protected]> * cargo fmt * disable cargo bench in dogstatd-client, we use criterion * Try using iterators in data-pipeline for metrics * bump ver * more metrics * Split dogstatsdaction type to simplify code * move to owned type for sidecar-ffi * remove unused import * more unused * Add variant check * add test that fails when dogstatsdAction is updated * fix tests * use either to simplify emit_metric code * resolve todo * Add docs and initial PR review comments * more docs and readme * add ticket for todo * PR comments * remove unused import * license header * rename flusher to client * fix names * Add dogstatd-client. * add more anyhow errors for easier debugging building dogstatsd client --------- Signed-off-by: Bob Weinand <[email protected]> Co-authored-by: Bob Weinand <[email protected]> Co-authored-by: Julio Gonzalez <[email protected]>
DataDog · Sep 24, 2024 · 829e93d · 829e93d
1 parent 59aacd2
commit 829e93d
Show file tree

Hide file tree

Showing 21 changed files with 659 additions and 292 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -31,6 +31,10 @@ component_management:
       name: ddtelemetry  # this is a display name, and can be changed freely
       paths:
         - ddtelemetry
+    - component_id: dogstatsd-client  # this is an identifier that should not be changed
+      name: dogstatsd-client  # this is a display name, and can be changed freely
+      paths:
+        - dogstatsd-client
     - component_id: ipc  # this is an identifier that should not be changed
       name: ipc  # this is a display name, and can be changed freely
       paths:

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -34,6 +34,7 @@ members = [
   "data-pipeline-ffi",
   "ddsketch",
   "tinybytes",
+  "dogstatsd-client",
 ]
 
 # https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2

diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml
@@ -1,4 +1,4 @@
-root_name: datadog-alloc, builder, build_common, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-ddsketch, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-obfuscation, datadog-trace-utils, ddcommon, tinybytes, ddcommon-ffi, datadog-crashtracker-ffi, datadog-crashtracker, ddtelemetry, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, dogstatsd, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent
+root_name: datadog-alloc, builder, build_common, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-ddsketch, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-obfuscation, datadog-trace-utils, ddcommon, tinybytes, dogstatsd-client, ddcommon-ffi, datadog-crashtracker-ffi, datadog-crashtracker, ddtelemetry, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, dogstatsd, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent
 third_party_libraries:
 - package_name: addr2line
   package_version: 0.21.0
@@ -10338,7 +10338,7 @@ third_party_libraries:
       OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
       SOFTWARE.
 - package_name: either
-  package_version: 1.10.0
+  package_version: 1.13.0
   repository: https://github.com/rayon-rs/either
   license: MIT OR Apache-2.0
   licenses:

diff --git a/data-pipeline/Cargo.toml b/data-pipeline/Cargo.toml
@@ -16,14 +16,16 @@ hyper = {version = "0.14", features = ["client"], default-features = false}
 log = "0.4"
 rmp-serde = "1.1.1"
 bytes = "1.4"
+either = "1.13.0"
 tokio = { version = "1.23", features = ["rt", "test-util", "time"], default-features = false }
 
 ddcommon = { path = "../ddcommon" }
 datadog-trace-protobuf = { path = "../trace-protobuf" }
 datadog-trace-utils = { path = "../trace-utils" }
 datadog-trace-normalization = { path = "../trace-normalization" }
+datadog-ddsketch = { path = "../ddsketch"}
+dogstatsd-client = { path = "../dogstatsd-client"}
 datadog-trace-obfuscation = { path = "../trace-obfuscation" }
-datadog-ddsketch = { path = "../ddsketch" }
 uuid = { version = "1.10.0", features = ["v4"] }
 tokio-util = "0.7.11"
 

diff --git a/data-pipeline/src/health_metrics.rs b/data-pipeline/src/health_metrics.rs
@@ -0,0 +1,14 @@
+// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+/// health_metrics holds data to emit info about the health of the data-pipeline
+
+pub(crate) const STAT_SEND_TRACES: &str = "datadog.libdatadog.send.traces";
+pub(crate) const STAT_SEND_TRACES_ERRORS: &str = "datadog.libdatadog.send.traces.errors";
+pub(crate) const STAT_DESER_TRACES: &str = "datadog.libdatadog.deser_traces";
+pub(crate) const STAT_DESER_TRACES_ERRORS: &str = "datadog.libdatadog.deser_traces.errors";
+pub(crate) const STAT_SER_TRACES_ERRORS: &str = "datadog.libdatadog.ser_traces.errors";
+
+pub(crate) enum HealthMetric {
+    Count(&'static str, i64),
+}
diff --git a/data-pipeline/src/lib.rs b/data-pipeline/src/lib.rs
@@ -6,6 +6,7 @@
 //! project at this state is to provide a basic API in order to test its viability and integration
 //! in different languages.
 
+mod health_metrics;
 #[allow(missing_docs)]
 pub mod span_concentrator;
 #[allow(missing_docs)]