diff --git a/.github/workflows/scripts/validate-helm-config-fields.sh b/.github/workflows/scripts/validate-helm-config-fields.sh index 4340413a907..0f87b047abb 100755 --- a/.github/workflows/scripts/validate-helm-config-fields.sh +++ b/.github/workflows/scripts/validate-helm-config-fields.sh @@ -656,12 +656,22 @@ bifrost: enabled: true config: service_name: "bifrost-dd" + ml_app: "bifrost-ml" agent_addr: "dd-agent:8126" + dogstatsd_addr: "dd-agent:8125" env: "staging" version: "1.0.0" custom_tags: team: "platform" + enable_metrics: true enable_traces: true + enable_llm_obs: true + disable_content_logging: false + agentless: true + api_key: "env.DD_API_KEY" + site: "datadoghq.com" + request_headers: + - "x-bf-session-id" custom: - name: "my-plugin" enabled: true @@ -727,11 +737,20 @@ assert_field_value 'plugins: otel insecure' '.plugins.[5].config.insecure' 'true # Datadog plugin assert_field_value 'plugins: datadog name' '.plugins.[6].name' '"datadog"' assert_field_value 'plugins: datadog service_name' '.plugins.[6].config.service_name' '"bifrost-dd"' +assert_field_value 'plugins: datadog ml_app' '.plugins.[6].config.ml_app' '"bifrost-ml"' assert_field_value 'plugins: datadog agent_addr' '.plugins.[6].config.agent_addr' '"dd-agent:8126"' +assert_field_value 'plugins: datadog dogstatsd_addr' '.plugins.[6].config.dogstatsd_addr' '"dd-agent:8125"' assert_field_value 'plugins: datadog env' '.plugins.[6].config.env' '"staging"' assert_field_value 'plugins: datadog version' '.plugins.[6].config.version' '"1.0.0"' assert_field 'plugins: datadog custom_tags' '.plugins.[6].config.custom_tags' +assert_field_value 'plugins: datadog enable_metrics' '.plugins.[6].config.enable_metrics' 'true' assert_field_value 'plugins: datadog enable_traces' '.plugins.[6].config.enable_traces' 'true' +assert_field_value 'plugins: datadog enable_llm_obs' '.plugins.[6].config.enable_llm_obs' 'true' +assert_field_value 'plugins: datadog disable_content_logging' '.plugins.[6].config.disable_content_logging' 'false' +assert_field_value 'plugins: datadog agentless' '.plugins.[6].config.agentless' 'true' +assert_field_value 'plugins: datadog api_key' '.plugins.[6].config.api_key' '"env.DD_API_KEY"' +assert_field_value 'plugins: datadog site' '.plugins.[6].config.site' '"datadoghq.com"' +assert_field 'plugins: datadog request_headers' '.plugins.[6].config.request_headers' # Custom plugin assert_field_value 'plugins: custom name' '.plugins.[7].name' '"my-plugin"' diff --git a/docs/enterprise/datadog-connector.mdx b/docs/enterprise/datadog-connector.mdx index f5ee90c4d93..b4a22b69da5 100644 --- a/docs/enterprise/datadog-connector.mdx +++ b/docs/enterprise/datadog-connector.mdx @@ -62,8 +62,8 @@ This mode requires an API key but simplifies deployment by eliminating the need |-------|------|----------|---------|-------------| | `service_name` | `string` | No | `bifrost` | Service name displayed in Datadog APM | | `ml_app` | `string` | No | (uses `service_name`) | ML application name for LLM Observability grouping | -| `agent_addr` | `string` | No | `localhost:8126` | Datadog Agent address (agent mode only) | -| `dogstatsd_addr` | `string` | No | `localhost:8125` | DogStatsD server address (agent mode only) | +| `agent_addr` | `string` | No | `localhost:8126` | Datadog Agent address (agent mode only, supports `env.VAR_NAME`) | +| `dogstatsd_addr` | `string` | No | `localhost:8125` | DogStatsD server address (agent mode only, supports `env.VAR_NAME`) | | `env` | `string` | No | - | Environment tag (e.g., `production`, `staging`) | | `version` | `string` | No | - | Service version tag | | `custom_tags` | `object` | No | - | Additional tags for all traces and metrics | @@ -71,16 +71,18 @@ This mode requires an API key but simplifies deployment by eliminating the need | `enable_traces` | `bool` | No | `true` | Enable APM traces | | `enable_llm_obs` | `bool` | No | `true` | Enable LLM Observability | | `agentless` | `bool` | No | `false` | Use agentless mode (direct API) | -| `api_key` | `EnvVar` | Agentless only | - | Datadog API key (supports `env.VAR_NAME`) | +| `api_key` | `string` | Agentless only | - | Datadog API key (supports `env.VAR_NAME`) | | `site` | `string` | No | `datadoghq.com` | Datadog site/region | ### Environment Variable Substitution -The `api_key` and `custom_tags` fields support environment variable substitution using the `env.` prefix: +The `api_key`, `agent_addr`, `dogstatsd_addr`, and `custom_tags` fields support environment variable substitution using the `env.` prefix: ```json { "api_key": "env.DD_API_KEY", + "agent_addr": "env.DD_AGENT_ADDR", + "dogstatsd_addr": "env.DD_DOGSTATSD_ADDR", "custom_tags": { "team": "env.TEAM_NAME", "cost_center": "env.COST_CENTER" @@ -354,11 +356,27 @@ The plugin emits the following metrics to Datadog: | `bifrost.tokens.input` | Counter | Input/prompt tokens consumed | provider, model | | `bifrost.tokens.output` | Counter | Output/completion tokens generated | provider, model | | `bifrost.tokens.total` | Counter | Total tokens (input + output) | provider, model | -| `bifrost.cost.usd` | Gauge | Request cost in USD | provider, model | +| `bifrost.request.cost.usd` | Distribution | Per-request cost in USD | provider, model | | `bifrost.cache.hits` | Counter | Cache hits | provider, model, cache_type | | `bifrost.stream.first_token_latency` | Histogram | Time to first token (streaming) | provider, model | | `bifrost.stream.inter_token_latency` | Histogram | Inter-token latency (streaming) | provider, model | +### Migrating from `bifrost.cost.usd` + +The cost metric was renamed from `bifrost.cost.usd` to `bifrost.request.cost.usd`, and its type changed from **Gauge** to **Distribution**. The gauge was last-write-wins per flush window, so concurrent requests with the same tags collapsed to a single value and no query could recover the true total spend. The new name is required because Datadog permanently associates a metric name with its first-seen type per organization — orgs that previously received the gauge cannot receive the same name as a distribution. + +**Affected assets:** any dashboards, monitors, saved views, or alerts that query `bifrost.cost.usd`. + +**To migrate:** + +1. Replace `bifrost.cost.usd` with `bifrost.request.cost.usd` in all queries. +2. Update aggregations for Distribution semantics — each sample is one request's cost: + - Total spend: `sum:bifrost.request.cost.usd{*}` (do **not** append `.as_count()` or `.rollup(sum)`; the `sum:` aggregator already returns the additive total) + - Per-request statistics: `avg:`, `max:`, or percentile aggregators +3. Recreate monitors and alerts on the new metric, adjusting thresholds if they assumed gauge behavior (the gauge systematically under-reported under concurrent load). + +`bifrost.cost.usd` stops receiving data once the upgrade completes; during a rolling deploy both metrics receive data, so update dashboards at or shortly after the upgrade. Historical gauge data remains queryable under the old name for Datadog's standard retention window. + ### Custom Tags All metrics include your configured `custom_tags` plus automatic tags for: @@ -366,6 +384,7 @@ All metrics include your configured `custom_tags` plus automatic tags for: - `model` - Model name - `request_type` - Type of request (chat, embedding, etc.) - `env` - Environment from configuration +- `bifrost_node` - Per-instance identity (`BIFROST_NODE_ID` if set, otherwise `hostname-pid`) --- diff --git a/helm-charts/bifrost/templates/_helpers.tpl b/helm-charts/bifrost/templates/_helpers.tpl index 04eb9669a0b..65056996d79 100644 --- a/helm-charts/bifrost/templates/_helpers.tpl +++ b/helm-charts/bifrost/templates/_helpers.tpl @@ -1185,9 +1185,15 @@ false {{- if $inputConfig.service_name }} {{- $_ := set $datadogConfig "service_name" $inputConfig.service_name }} {{- end }} +{{- if $inputConfig.ml_app }} +{{- $_ := set $datadogConfig "ml_app" $inputConfig.ml_app }} +{{- end }} {{- if $inputConfig.agent_addr }} {{- $_ := set $datadogConfig "agent_addr" $inputConfig.agent_addr }} {{- end }} +{{- if $inputConfig.dogstatsd_addr }} +{{- $_ := set $datadogConfig "dogstatsd_addr" $inputConfig.dogstatsd_addr }} +{{- end }} {{- if $inputConfig.env }} {{- $_ := set $datadogConfig "env" $inputConfig.env }} {{- end }} @@ -1197,9 +1203,30 @@ false {{- if $inputConfig.custom_tags }} {{- $_ := set $datadogConfig "custom_tags" $inputConfig.custom_tags }} {{- end }} +{{- if hasKey $inputConfig "enable_metrics" }} +{{- $_ := set $datadogConfig "enable_metrics" $inputConfig.enable_metrics }} +{{- end }} {{- if hasKey $inputConfig "enable_traces" }} {{- $_ := set $datadogConfig "enable_traces" $inputConfig.enable_traces }} {{- end }} +{{- if hasKey $inputConfig "enable_llm_obs" }} +{{- $_ := set $datadogConfig "enable_llm_obs" $inputConfig.enable_llm_obs }} +{{- end }} +{{- if hasKey $inputConfig "disable_content_logging" }} +{{- $_ := set $datadogConfig "disable_content_logging" $inputConfig.disable_content_logging }} +{{- end }} +{{- if hasKey $inputConfig "agentless" }} +{{- $_ := set $datadogConfig "agentless" $inputConfig.agentless }} +{{- end }} +{{- if $inputConfig.api_key }} +{{- $_ := set $datadogConfig "api_key" $inputConfig.api_key }} +{{- end }} +{{- if $inputConfig.site }} +{{- $_ := set $datadogConfig "site" $inputConfig.site }} +{{- end }} +{{- if $inputConfig.request_headers }} +{{- $_ := set $datadogConfig "request_headers" $inputConfig.request_headers }} +{{- end }} {{- if $inputConfig.plugin_span_filter }} {{- $_ := set $datadogConfig "plugin_span_filter" $inputConfig.plugin_span_filter }} {{- end }} @@ -1408,6 +1435,10 @@ Call this template at the beginning of deployment/stateful templates {{- if and .Values.bifrost.plugins.datadog.enabled (hasKey .Values.bifrost.plugins.datadog "version") (gt (int .Values.bifrost.plugins.datadog.version) 32767) }} {{- fail "ERROR: bifrost.plugins.datadog.version must be <= 32767." }} {{- end }} +{{- $ddCfg := (.Values.bifrost.plugins.datadog.config | default dict) }} +{{- if and .Values.bifrost.plugins.datadog.enabled $ddCfg.agentless (not $ddCfg.api_key) }} +{{- fail "ERROR: bifrost.plugins.datadog.config.api_key is required when bifrost.plugins.datadog.config.agentless is true." }} +{{- end }} {{- if and .Values.bifrost.plugins.bigquery.enabled (hasKey .Values.bifrost.plugins.bigquery "version") (lt (int .Values.bifrost.plugins.bigquery.version) 1) }} {{- fail "ERROR: bifrost.plugins.bigquery.version must be >= 1. Bump to >1 to force DB-backed plugin config updates." }} {{- end }} diff --git a/helm-charts/bifrost/values.schema.json b/helm-charts/bifrost/values.schema.json index 9f959f2cb00..40e12905fa9 100644 --- a/helm-charts/bifrost/values.schema.json +++ b/helm-charts/bifrost/values.schema.json @@ -872,14 +872,27 @@ "enabled": { "type": "boolean" }, + "version": { + "type": "integer", + "minimum": 1 + }, "config": { "type": "object", "properties": { "service_name": { "type": "string" }, + "ml_app": { + "type": "string", + "description": "ML application name for Datadog LLM Observability grouping (defaults to service_name)" + }, "agent_addr": { - "type": "string" + "type": "string", + "description": "Datadog Agent address for APM traces (agent mode only). Supports env.VAR_NAME prefix for environment variable substitution (e.g. env.DD_AGENT_ADDR)" + }, + "dogstatsd_addr": { + "type": "string", + "description": "DogStatsD server address for metrics (agent mode only). Supports env.VAR_NAME prefix for environment variable substitution (e.g. env.DD_DOGSTATSD_ADDR)" }, "env": { "type": "string" @@ -888,10 +901,46 @@ "type": "string" }, "custom_tags": { - "type": "object" + "type": "object", + "description": "Custom tags for Datadog metrics and traces. Values support env.VAR_NAME prefix for environment variable substitution", + "additionalProperties": { + "type": "string" + } + }, + "enable_metrics": { + "type": "boolean", + "description": "Enable Datadog metrics emission (default: true)" }, "enable_traces": { - "type": "boolean" + "type": "boolean", + "description": "Enable Datadog APM traces (default: true)" + }, + "enable_llm_obs": { + "type": "boolean", + "description": "Enable Datadog LLM Observability (default: true)" + }, + "disable_content_logging": { + "type": "boolean", + "description": "When true, sensitive content (prompts, completions, embeddings) is excluded from traces (default: false)" + }, + "agentless": { + "type": "boolean", + "description": "Use agentless mode to send data directly to Datadog APIs instead of a local agent. Requires api_key (default: false)" + }, + "api_key": { + "type": "string", + "description": "Datadog API key, required for agentless mode. Supports env.VAR_NAME prefix for environment variable substitution (e.g. env.DD_API_KEY)" + }, + "site": { + "type": "string", + "description": "Datadog site/region for agentless mode (e.g. datadoghq.com, datadoghq.eu)" + }, + "request_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Request header names to capture as Datadog span tags" }, "plugin_span_filter": { "$ref": "#/$defs/pluginSpanFilter" diff --git a/helm-charts/bifrost/values.yaml b/helm-charts/bifrost/values.yaml index 159fd6ebd6a..ff605c6fc93 100644 --- a/helm-charts/bifrost/values.yaml +++ b/helm-charts/bifrost/values.yaml @@ -515,11 +515,24 @@ bifrost: version: 1 config: service_name: "bifrost" + # Datadog Agent address. Supports env.VAR_NAME references — e.g. set + # agent_addr: "env.DD_AGENT_ADDR" and inject DD_AGENT_ADDR via the + # top-level `env:` (e.g. from status.hostIP for a node-local agent DaemonSet). agent_addr: "localhost:8126" + # dogstatsd_addr: "localhost:8125" # DogStatsD address (supports env.VAR_NAME) env: "" version: "" custom_tags: {} enable_traces: true + # ml_app: "" # ML app name for LLM Observability (defaults to service_name) + # enable_metrics: true + # enable_llm_obs: true + # disable_content_logging: false + # request_headers: [] # Header name patterns (exact or wildcard like "x-custom-*") + # Agentless mode (direct to Datadog API, no local agent): + # agentless: true + # api_key: "env.DD_API_KEY" # Required for agentless mode (supports env.VAR_NAME) + # site: "datadoghq.com" # Datadog site/region (e.g. datadoghq.eu) # plugin_span_filter: # Optional: filter which plugin hook spans are exported # mode: "exclude" # "include" or "exclude" # plugins: ["logging"] diff --git a/transports/config.schema.json b/transports/config.schema.json index 6d7daf9e231..d9c2a5de4a8 100644 --- a/transports/config.schema.json +++ b/transports/config.schema.json @@ -1685,11 +1685,20 @@ "description": "Name of the service to report to Datadog", "default": "bifrost" }, + "ml_app": { + "type": "string", + "description": "ML application name for LLM Observability grouping (defaults to service_name)" + }, "agent_addr": { "type": "string", - "description": "Address of the Datadog Agent for APM traces", + "description": "Address of the Datadog Agent for APM traces (agent mode only, can use env. prefix)", "default": "localhost:8126" }, + "dogstatsd_addr": { + "type": "string", + "description": "Address of the DogStatsD server for metrics (agent mode only, can use env. prefix)", + "default": "localhost:8125" + }, "env": { "type": "string", "description": "Environment tag (e.g., production, staging)" @@ -1703,17 +1712,68 @@ "additionalProperties": { "type": "string" }, - "description": "Additional tags to add to all traces and metrics" + "description": "Additional tags to add to all traces and metrics (values can use env. prefix)" + }, + "enable_metrics": { + "type": "boolean", + "description": "Enable metrics emission (default: true)", + "default": true }, "enable_traces": { "type": "boolean", "description": "Enable APM traces (default: true)", "default": true }, + "enable_llm_obs": { + "type": "boolean", + "description": "Enable LLM Observability (default: true)", + "default": true + }, + "disable_content_logging": { + "type": "boolean", + "description": "Disable logging of message content to Datadog (default: false)", + "default": false + }, + "agentless": { + "type": "boolean", + "description": "Use agentless mode to send data directly to Datadog APIs (default: false)", + "default": false + }, + "api_key": { + "type": "string", + "description": "Datadog API key, required for agentless mode (can use env. prefix)" + }, + "site": { + "type": "string", + "description": "Datadog site/region (e.g., datadoghq.com, datadoghq.eu)", + "default": "datadoghq.com" + }, + "request_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Request header names to capture as span tags" + }, "plugin_span_filter": { "$ref": "#/$defs/plugin_span_filter" } }, + "allOf": [ + { + "if": { + "properties": { + "agentless": { + "const": true + } + }, + "required": ["agentless"] + }, + "then": { + "required": ["api_key"] + } + } + ], "additionalProperties": false } }