Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 101 additions & 16 deletions crates/ourios-core/src/otlp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -957,14 +957,21 @@ impl Default for TenantId {
}

/// Lenient OTLP/JSON parsing: a workaround for `opentelemetry-proto`'s
/// `with-serde` deserialiser rejecting proto3-JSON's two valid
/// encodings of an **unset** `AnyValue` — `{}` (a message with no
/// fields set; the `OTel` Collector's file exporter emits it for
/// empty-body event records) and `null` (what `with-serde`'s own
/// serialiser emits for `value: None`) — with "Invalid data for
/// Value, no known keys found". Tracked upstream; until the fix
/// ships, both OTLP/JSON entry points (the receiver's
/// `application/json` transport and the bench corpus loader) parse
/// `with-serde` deserialiser rejecting proto3-JSON's valid encodings
/// of an **unset** `AnyValue` — `{}` (a message with no fields set;
/// the `OTel` Collector's file exporter emits it for empty-body event
/// records), `null` (what `with-serde`'s own serialiser emits for
/// `value: None`), and a field set to `null` such as
/// `{"intValue":null}` / `{"doubleValue":null}` (proto3-JSON maps a
/// `null` field to its default — for a `oneof`, unset; real exporters
/// emit this, e.g. the Vercel AI SDK for non-finite token counts,
/// upstream opentelemetry-rust#3603). The broken deserialiser fails
/// all three with "Invalid data for Value, no known keys found" (or,
/// for the null field, an untagged-enum-variant error). Tracked
/// upstream (opentelemetry-rust#3595 for `{}`, #3603 for the null
/// field); until both ship, both OTLP/JSON entry points (the
/// receiver's `application/json` transport and the bench corpus
/// loader) parse
/// through here.
///
/// Semantics: the direct parse runs first, so the hot path is
Expand Down Expand Up @@ -1031,8 +1038,10 @@ pub mod lenient_json {
}

/// Remove `"body"` / `"value"` keys whose value encodes an unset
/// `AnyValue` (`{}` or `null`), recursively. Returns whether the
/// tree changed. The two key names are unambiguous in the OTLP/JSON
/// `AnyValue` — `null`, the empty object `{}`, or an object whose
/// recognised field is `null` (`{"intValue":null}`) — recursively.
/// Returns whether the tree changed. The two key names are
/// unambiguous in the OTLP/JSON
/// log tree: user data only ever appears as the *values of* `key` /
/// `value` / `body` fields, never as sibling object keys, so no
/// user-controlled content can be misread as one of these
Expand All @@ -1044,7 +1053,38 @@ pub mod lenient_json {
for slot in ["body", "value"] {
let is_unset = match map.get(slot) {
Some(serde_json::Value::Null) => true,
Some(serde_json::Value::Object(inner)) => inner.is_empty(),
// An `AnyValue` object is unset when every entry is a
// *recognised* `oneof` field name whose value is `null`
// — proto3-JSON maps a `null` field to its default, and
// for a `oneof` that is unset. `{"intValue":null}` /
// `{"doubleValue":null}` (real exporter traffic — the
// Vercel AI SDK emits the latter for non-finite token
// counts). `all` over an empty map is vacuously true, so
// this subsumes `{}`. The recognised-key requirement is
// load-bearing: it keeps the strip narrow so it only
// ever removes what is genuinely an unset value, never
// papering over a malformed one. Two things stay
// untouched: a present non-null value like
// `{"stringValue":""}` (an empty string is a set value
// the spec says MUST be stored), and an *unknown* key
// even with a null value like `{"bogusValue":null}`
// (which stays a parse error — the unknown-key rejection
// the direct parse already enforces).
Some(serde_json::Value::Object(inner)) => {
inner.iter().all(|(key, value)| {
value.is_null()
&& matches!(
key.as_str(),
"stringValue"
| "boolValue"
| "intValue"
| "doubleValue"
| "bytesValue"
| "arrayValue"
| "kvlistValue"
)
})
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
_ => false,
};
if is_unset {
Expand Down Expand Up @@ -1252,11 +1292,21 @@ mod tests {
fn lenient_json_does_not_loosen_unknown_key_rejection() {
use opentelemetry_proto::tonic::logs::v1::LogsData;
// An AnyValue object with keys — just no KNOWN key — stays an
// error: the retry only rewrites genuinely EMPTY encodings, it
// must not paper over malformed values.
let doc =
r#"{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"body":{"bogusValue":1}}]}]}]}"#;
assert!(lenient_json::from_slice::<LogsData>(doc.as_bytes()).is_err());
// error: the retry only rewrites genuinely unset encodings, it
// must not paper over malformed values. This holds whether the
// unknown key's value is non-null (`{"bogusValue":1}`) OR null
// (`{"bogusValue":null}`) — the null-field relaxation only
// strips objects whose keys are all *recognised* oneof fields,
// so an unknown key stays a parse error even when null-valued.
for doc in [
r#"{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"body":{"bogusValue":1}}]}]}]}"#,
r#"{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"body":{"bogusValue":null}}]}]}]}"#,
] {
assert!(
lenient_json::from_slice::<LogsData>(doc.as_bytes()).is_err(),
"unknown key must stay an error: {doc}"
);
}
}

#[test]
Expand All @@ -1270,4 +1320,39 @@ mod tests {
Some(AvValue::StringValue(s)) if s == "hello"
));
}

#[test]
fn lenient_json_strips_a_null_field_but_keeps_a_meaningful_empty_string() {
use opentelemetry_proto::tonic::logs::v1::LogsData;
// A recognised AnyValue field set to `null` — proto3-JSON's
// "null = field default" (for a `oneof`, unset). Real exporters
// emit this: the Vercel AI SDK writes `doubleValue: null` for
// non-finite token counts (upstream opentelemetry-rust#3603). An
// empty *string*, by contrast, is a SET value the spec says MUST
// be stored — so the relaxation must strip the former without
// touching the latter. One doc exercises both through the retry.
let doc = r#"{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"body":{"intValue":null},"attributes":[{"key":"tokens","value":{"doubleValue":null}},{"key":"note","value":{"stringValue":""}}]}]}]}]}"#;
let (parsed, lenient): (LogsData, bool) =
lenient_json::from_slice_flagged(doc.as_bytes()).expect("lenient parse");
assert!(
lenient,
"the null fields only parse via the lenient retry today"
);
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
let record = &parsed.resource_logs[0].scope_logs[0].log_records[0];
let unset = |v: &Option<opentelemetry_proto::tonic::common::v1::AnyValue>| {
v.as_ref().and_then(|av| av.value.as_ref()).is_none()
};
assert!(unset(&record.body), "null intValue field -> unset body");
assert!(
unset(&record.attributes[0].value),
"null doubleValue field -> unset value"
);
assert!(
matches!(
record.attributes[1].value.as_ref().and_then(|v| v.value.as_ref()),
Some(AvValue::StringValue(s)) if s.is_empty()
),
"an empty string is a SET value and must survive the null-field strip"
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -330,3 +330,34 @@ fn rfc0003_6_unset_any_value_decodes_equivalently_across_transports() {
}
assert_eq!(pb_rec.event_name, js_rec.event_name);
}

/// RFC0003.6 companion — a `null`-valued `AnyValue` *field*
/// (`{"intValue":null}` / `{"doubleValue":null}`) is proto3-JSON's
/// encoding of a field default (for a `oneof`, unset). Real exporters
/// emit it: the Vercel AI SDK writes `doubleValue: null` for non-finite
/// token counts (upstream opentelemetry-rust#3603) — exactly the
/// AI-agent traffic RFC 0037 targets. The production JSON receive path
/// (`decode_json`) must accept it via the lenient shim rather than
/// reject the whole batch, decoding it to the same absent state as `{}`.
/// This is also the null-field arm of the shim-retirement flip signal:
/// removal needs BOTH #3595 (`{}`) and #3603 (the null field) released.
Comment thread
jensholdgaard marked this conversation as resolved.
Resolved
#[test]
fn rfc0003_6_null_valued_anyvalue_field_decodes_via_lenient_shim() {
let json = br#"{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"severityNumber":9,"body":{"intValue":null},"attributes":[{"key":"gen_ai.usage.output_tokens","value":{"doubleValue":null}}],"eventName":"gen_ai.client.inference.operation.details"}]}]}]}"#;
let (decoded, lenient) =
decode_json(json).expect("a null-field unset AnyValue must decode, not reject the batch");
assert!(
lenient,
"the null field only parses via the lenient retry today — the null-field arm of \
the flip signal (direct parse succeeding makes this false; needs upstream #3603)",
);
let rec = &decoded.resource_logs[0].scope_logs[0].log_records[0];
assert_eq!(
rec.body, None,
"a null intValue field decodes to an absent body"
);
assert_eq!(
rec.attributes[0].value, None,
"a null doubleValue field decodes to an absent value",
);
}