fix(parsers): DeepSeek V4.1 grammar and recovery

conformance/tests/capture_cross_version.rs

@@ -45,9 +45,7 @@ use std::path::{Path, PathBuf};
 use dynamo_parsers::{ReasoningParser, ReasoningParserType};
 use dynamo_parsers_v2::create_tool_parser_for_family;
 #[cfg(not(conformance_split_only))]
-use dynamo_parsers_v2::{
-    REGISTERED_UNIFIED_FAMILIES, UnifiedEvent, assemble, create_unified_parser_for_family,
-};
+use dynamo_parsers_v2::{REGISTERED_UNIFIED_FAMILIES, create_unified_parser_for_family};
 use serde_json::{Value, json};
 
 mod common;
@@ -596,11 +594,6 @@ fn split_path_chunks_scheduled(
     Ok(rows)
 }
 
-#[cfg(not(conformance_split_only))]
-fn ev_to_yaml(ev: &UnifiedEvent) -> serde_yaml::Value {
-    common::unified_event_to_yaml_value(ev)
-}
-
 /// Fold per-chunk deltas into assembled events, mirroring the page's `_assemble_stream`:
 /// consecutive same-kind text/reasoning runs concatenate, and a tool_call delta carrying
 /// a name opens a call while later nameless fragments append to its argument string.
@@ -741,7 +734,7 @@ fn native_capture(
         .iter()
         .map(|row| row.iter().map(delta_to_yaml).collect())
         .collect();
-    Ok((rows, assemble(&deltas).iter().map(ev_to_yaml).collect()))
+    Ok((rows, capture::assembled_native_capture(&deltas).0))
 }
 
 fn capture_case_with_chunks(
@@ -816,14 +809,41 @@ fn split_capture_assembles_reasoning_over_the_whole_input() {
     assert_eq!(assembled, want);
 }
 
+#[cfg(all(
+    not(conformance_split_only),
+    not(conformance_legacy_indexed_assembly),
+    not(conformance_legacy_delta)
+))]
+#[test]
+fn native_capture_producer_preserves_raw_numeric_arguments() {
+    let raw = r#"{"fraction":9007199254740992.5,"tiny":1e-400,"huge":1e400}"#;
+    let input = format!(
+        "<｜DSML｜ calls><｜DSML｜ invoke name=\"inspect\"><｜DSML｜ parameter name=\"value\" string=\"false\">{raw}</｜DSML｜ parameter></｜DSML｜ invoke></｜DSML｜ calls>"
+    );
+    let mut parser = create_unified_parser_for_family("deepseek_v41", &[]).unwrap();
+    let (_, events) =
+        native_capture(&mut parser, &chunk_input(&input), &common::Init::default()).unwrap();
+    assert_eq!(events.len(), 1);
+    assert_eq!(events[0]["name"].as_str(), Some("inspect"));
+    let actual = events[0]["arguments"]
+        .as_str()
+        .expect("raw capture arguments");
+    let expected = format!(r#"{{"value":{raw}}}"#);
+    assert_eq!(
+        common::canonical_argument_json(actual),
+        common::canonical_argument_json(&expected)
+    );
+}
+
 #[test]
 fn json_to_yaml_preserves_large_integers_and_yaml_incompatible_floats() {
-    let value: Value = serde_json::from_str(r#"{"value":9007199254740993,"tiny":1e-400}"#).unwrap();
+    let value: Box<serde_json::value::RawValue> =
+        serde_json::from_str(r#"{"value":9007199254740993,"tiny":1e-400}"#).unwrap();
     let yaml = common::json_to_yaml_value(&value);
     assert_eq!(yaml["value"].as_u64(), Some(9_007_199_254_740_993));
     assert_eq!(yaml["tiny"].as_str(), Some("1e-400"));
     for raw in ["0e1", "0.0e-400"] {
-        let value: Value = serde_json::from_str(raw).unwrap();
+        let value: Box<serde_json::value::RawValue> = serde_json::from_str(raw).unwrap();
         assert!(common::json_to_yaml_value(&value).is_number());
     }
 }
@@ -837,11 +857,23 @@ fn captured_event_arguments_round_trip_without_yaml_number_conversion() {
         r#""null""#,
         r#""[1]""#,
     ] {
-        let arguments: Value = serde_json::from_str(raw).unwrap();
-        let event = json!({"kind": "tool_call", "name": "inspect", "arguments": arguments});
+        #[derive(serde::Serialize)]
+        struct Event<'a> {
+            kind: &'a str,
+            name: &'a str,
+            arguments: Box<serde_json::value::RawValue>,
+        }
+        let event = Event {
+            kind: "tool_call",
+            name: "inspect",
+            arguments: serde_json::from_str(raw).unwrap(),
+        };
         let yaml = common::unified_event_to_yaml_value(&event);
         let captured = yaml["arguments"].as_str().expect("raw JSON arguments");
-        assert_eq!(serde_json::from_str::<Value>(captured).unwrap(), arguments);
+        assert_eq!(
+            common::canonical_argument_json(captured),
+            common::canonical_argument_json(raw)
+        );
     }
 }
 
@@ -1137,9 +1169,21 @@ fn family_jobs_replay_terminal_tool_calls_without_accepting_chunk_metadata() {
         let record = &results["gemma4"][reason];
         assert!(record.get("unavailable").is_none(), "{reason}: {record}");
         let mut expected = case["golden"]["assembled"].clone();
-        for event in expected.as_array_mut().unwrap() {
-            if event["kind"] == "tool_call" && !event["arguments"].is_string() {
-                event["arguments"] = json!(serde_json::to_string(&event["arguments"]).unwrap());
+        for (index, event) in expected.as_array_mut().unwrap().iter_mut().enumerate() {
+            if event["kind"] == "tool_call" {
+                let expected_arguments = event["arguments"]
+                    .as_str()
+                    .map(str::to_owned)
+                    .unwrap_or_else(|| serde_json::to_string(&event["arguments"]).unwrap());
+                let actual_arguments = record["assembled"][index]["arguments"]
+                    .as_str()
+                    .expect("captured raw JSON arguments");
+                assert_eq!(
+                    common::canonical_argument_json(actual_arguments),
+                    common::canonical_argument_json(&expected_arguments),
+                    "{reason}: tool arguments"
+                );
+                event["arguments"] = json!(actual_arguments);
             }
         }
         assert_eq!(record["assembled"], expected);

conformance/tests/common/mod.rs

@@ -135,8 +135,8 @@ mod json_comparison_tests {
             (r#"{"value":1e-400}"#, r#"{"value":0}"#, false),
             ("[1,2]", "[2,1]", false),
         ] {
-            let left: serde_json::Value = serde_json::from_str(left).unwrap();
-            let right: serde_json::Value = serde_json::from_str(right).unwrap();
+            let left: Box<serde_json::value::RawValue> = serde_json::from_str(left).unwrap();
+            let right: Box<serde_json::value::RawValue> = serde_json::from_str(right).unwrap();
             assert_eq!(super::json_values_equal(&left, &right), equal);
         }
     }
@@ -152,53 +152,70 @@ pub fn decoded_golden_arguments(arguments: &serde_json::Value) -> serde_json::Va
 }
 
 pub fn json_to_yaml_value<T: serde::Serialize>(value: &T) -> serde_yaml::Value {
-    fn convert(value: serde_json::Value) -> serde_yaml::Value {
-        match value {
-            serde_json::Value::Null => serde_yaml::Value::Null,
-            serde_json::Value::Bool(value) => serde_yaml::Value::Bool(value),
-            serde_json::Value::Number(number) => {
-                let lexeme = number.to_string();
-                let nonzero_digit = lexeme
-                    .split(['e', 'E'])
-                    .next()
-                    .expect("JSON number has a mantissa")
-                    .bytes()
-                    .any(|byte| byte.is_ascii_digit() && byte != b'0');
-                let yaml_incompatible = number
-                    .as_f64()
-                    .is_none_or(|value| !value.is_finite() || value == 0.0 && nonzero_digit);
-                if yaml_incompatible {
-                    serde_yaml::Value::String(lexeme)
-                } else {
-                    serde_yaml::from_str(&lexeme).expect("JSON number parses as YAML")
-                }
+    fn convert(value: &serde_json::value::RawValue) -> serde_yaml::Value {
+        let raw = value.get();
+        match raw.as_bytes().first() {
+            Some(b'n') => serde_yaml::Value::Null,
+            Some(b't') => serde_yaml::Value::Bool(true),
+            Some(b'f') => serde_yaml::Value::Bool(false),
+            Some(b'"') => serde_yaml::Value::String(serde_json::from_str(raw).unwrap()),
+            Some(b'[') => {
+                let values: Vec<Box<serde_json::value::RawValue>> =
+                    serde_json::from_str(raw).unwrap();
+                serde_yaml::Value::Sequence(values.iter().map(|value| convert(value)).collect())
             }
-            serde_json::Value::String(value) => serde_yaml::Value::String(value),
-            serde_json::Value::Array(values) => {
-                serde_yaml::Value::Sequence(values.into_iter().map(convert).collect())
+            Some(b'{') => {
+                let values: BTreeMap<String, Box<serde_json::value::RawValue>> =
+                    serde_json::from_str(raw).unwrap();
+                serde_yaml::Value::Mapping(
+                    values
+                        .into_iter()
+                        .map(|(key, value)| (serde_yaml::Value::String(key), convert(&value)))
+                        .collect(),
+                )
             }
-            serde_json::Value::Object(values) => serde_yaml::Value::Mapping(
-                values
-                    .into_iter()
-                    .map(|(key, value)| (serde_yaml::Value::String(key), convert(value)))
-                    .collect(),
-            ),
+            Some(b'-' | b'0'..=b'9') => {
+                // YAML scalars use bounded numbers; preserve tokens whose value
+                // changes when that representation is serialized back to JSON.
+                let yaml = serde_yaml::from_str::<serde_yaml::Value>(raw)
+                    .ok()
+                    .filter(serde_yaml::Value::is_number)
+                    .or_else(|| {
+                        raw.parse::<f64>()
+                            .ok()
+                            .and_then(|value| serde_yaml::to_value(value).ok())
+                    });
+                if let Some(yaml) = yaml
+                    && let Ok(round_trip) = serde_json::to_string(&yaml)
+                    && canonical_argument_json(&round_trip) == canonical_argument_json(raw)
+                {
+                    return yaml;
+                }
+                serde_yaml::Value::String(raw.to_string())
+            }
+            _ => unreachable!("RawValue only accepts JSON"),
         }
     }
 
-    convert(serde_json::to_value(value).expect("value serializes as JSON"))
+    let raw = serde_json::value::to_raw_value(value).expect("value serializes as JSON");
+    convert(&raw)
 }
 
 pub fn unified_event_to_yaml_value<T: serde::Serialize>(event: &T) -> serde_yaml::Value {
-    let mut event = serde_json::to_value(event).expect("event serializes as JSON");
-    if event.get("kind").and_then(serde_json::Value::as_str) == Some("tool_call")
-        && let Some(arguments) = event.get_mut("arguments")
-    {
+    let event = serde_json::value::to_raw_value(event).expect("event serializes as JSON");
+    if !event.get().starts_with('{') {
+        return json_to_yaml_value(&event);
+    }
+    let mut event: BTreeMap<String, Box<serde_json::value::RawValue>> =
+        serde_json::from_str(event.get()).expect("event object");
+    let tool_call = event.get("kind").is_some_and(|kind| {
+        serde_json::from_str::<String>(kind.get()).is_ok_and(|kind| kind == "tool_call")
+    });
+    if tool_call && let Some(arguments) = event.get_mut("arguments") {
         // Preserve the whole JSON payload: YAML float scalars round decimals,
         // and quoting individual numbers would change their argument types.
-        *arguments = serde_json::Value::String(
-            serde_json::to_string(arguments).expect("tool arguments serialize as JSON"),
-        );
+        *arguments = serde_json::value::to_raw_value(&arguments.get())
+            .expect("tool arguments serialize as JSON text");
     }
     json_to_yaml_value(&event)
 }

conformance/tests/common/unified_capture.rs

@@ -132,6 +132,82 @@ pub fn native_capture(
     Ok(rows)
 }
 
+/// One raw argument projection for current capture production and report replay.
+#[cfg(not(conformance_split_only))]
+pub fn assembled_native_capture(
+    deltas: &[NativeEvent],
+) -> (
+    Vec<serde_yaml::Value>,
+    std::collections::BTreeMap<usize, String>,
+) {
+    #[cfg(any(conformance_legacy_indexed_assembly, conformance_legacy_delta))]
+    {
+        return (
+            dynamo_parsers_v2::assemble(deltas)
+                .iter()
+                .map(super::unified_event_to_yaml_value)
+                .collect(),
+            std::collections::BTreeMap::new(),
+        );
+    }
+    #[cfg(not(any(conformance_legacy_indexed_assembly, conformance_legacy_delta)))]
+    {
+        let raw = dynamo_parsers_v2::tool_arguments_raw(deltas);
+        let events = dynamo_parsers_v2::assemble_with_tool_indices(deltas)
+            .into_iter()
+            .map(|(index, event)| {
+                let mut yaml = super::unified_event_to_yaml_value(&event);
+                if let Some(index) = index {
+                    let arguments = raw.get(&index).expect("assembled call has raw arguments");
+                    if serde_json::value::RawValue::from_string(arguments.clone()).is_ok() {
+                        yaml["arguments"] = serde_yaml::Value::String(arguments.clone());
+                    }
+                }
+                yaml
+            })
+            .collect();
+        (events, raw)
+    }
+}
+
+#[cfg(all(
+    not(conformance_split_only),
+    not(conformance_legacy_indexed_assembly),
+    not(conformance_legacy_delta)
+))]
+#[test]
+fn native_assembly_keeps_raw_numbers_and_sparse_surviving_indices() {
+    let call = |tool_index, name: Option<&str>, arguments: &str, complete| {
+        NativeEvent::ToolCall(dynamo_parsers_v2::ToolCallDelta {
+            tool_index,
+            name: name.map(str::to_owned),
+            arguments: arguments.into(),
+            complete,
+        })
+    };
+    let raw = r#"{"fraction":9007199254740992.5,"tiny":1e-400,"huge":1e400}"#;
+    let (events, arguments) = assembled_native_capture(&[
+        call(2, Some("drop"), "{", false),
+        call(9, Some("inspect"), raw, true),
+        NativeEvent::Text("between".into()),
+        call(5, Some("other"), r#"{"n":9007199254740993}"#, true),
+        call(7, Some("malformed"), "{bad}", true),
+    ]);
+    assert_eq!(arguments.get(&9).map(String::as_str), Some(raw));
+    assert_eq!(events.len(), 4);
+    assert_eq!(events[0]["name"].as_str(), Some("inspect"));
+    assert_eq!(events[0]["arguments"].as_str(), Some(raw));
+    assert_eq!(events[1]["kind"].as_str(), Some("text"));
+    assert_eq!(events[1]["text"].as_str(), Some("between"));
+    assert_eq!(events[2]["name"].as_str(), Some("other"));
+    assert_eq!(
+        events[2]["arguments"].as_str(),
+        Some(r#"{"n":9007199254740993}"#)
+    );
+    assert_eq!(events[3]["name"].as_str(), Some("malformed"));
+    assert_eq!(events[3]["arguments"].as_str(), Some("{}"));
+}
+
 #[test]
 fn authored_and_recorded_schedules_preserve_utf8_and_empty_chunks() {
     let input = "<call>éCafé</call>";

conformance/tests/unified_parity.rs

@@ -831,12 +831,25 @@ fn deepseek_tool_adapter_matches_both_native_golden_corpora() {
             *coverage.entry(file.family.clone()).or_default() += 1;
             for (label, chunks) in splittings(&case.input) {
                 let case_tools = common::unified_tools_for_schemas(case.tools.as_ref());
-                let native = events(&file.family, &chunks, &case.init, &case_tools).map(|events| {
-                    events
-                        .into_iter()
-                        .filter(|event| matches!(event, UnifiedEvent::ToolCall { .. }))
-                        .collect::<Vec<_>>()
-                });
+                let native = events_and_raw(&file.family, &chunks, &case.init, &case_tools).map(
+                    |(events, raw)| {
+                        let names = events
+                            .into_iter()
+                            .filter_map(|event| match event {
+                                UnifiedEvent::ToolCall { name, .. } => Some(name),
+                                _ => None,
+                            })
+                            .collect::<Vec<_>>();
+                        assert_eq!(names.len(), raw.len(), "native raw argument inventory");
+                        names
+                            .into_iter()
+                            .zip(raw.into_values())
+                            .map(|(name, arguments)| {
+                                (name, common::canonical_argument_json(&arguments))
+                            })
+                            .collect::<Vec<_>>()
+                    },
+                );
                 let mut parser = create_tool_parser_for_family("deepseek_v4", &case_tools).unwrap();
                 let mut result = ToolParseResult::default();
                 let mut failure = None;
@@ -861,9 +874,11 @@ fn deepseek_tool_adapter_matches_both_native_golden_corpora() {
                             .coalesce_calls()
                             .calls
                             .into_iter()
-                            .map(|call| UnifiedEvent::ToolCall {
-                                name: call.name.unwrap(),
-                                arguments: serde_json::from_str(&call.arguments).unwrap(),
+                            .map(|call| {
+                                (
+                                    call.name.unwrap(),
+                                    common::canonical_argument_json(&call.arguments),
+                                )
                             })
                             .collect::<Vec<_>>())
                     },