diff --git a/conformance/fixtures-manifest.json b/conformance/fixtures-manifest.json index c04dba375..ecfded68c 100644 --- a/conformance/fixtures-manifest.json +++ b/conformance/fixtures-manifest.json @@ -1,6 +1,6 @@ { - "snapshot": "20260827_110017", - "created_pt": "2026-08-27 11:00:17 America/Los_Angeles", + "snapshot": "20260828_105445", + "created_pt": "2026-08-28 10:54:45 America/Los_Angeles", "crates": { "dynamo-parsers": "8.2.0", "dynamo-parsers-v2": "0.3.4" @@ -230,16 +230,36 @@ "sha256": "ab6021d70826d91174a05a6bf143d7661502991440aace22983ba014b9475f4d", "size": 13798 }, + { + "path": "unified/dynamo_v2-0.3.4+pr166.patch1.tar.gz", + "sha256": "0ef7bbd599d848112002c934105386c58e53ba19d8692bc85e9033c639f63f56", + "size": 486 + }, + { + "path": "unified/dynamo_v2-0.3.4+pr166.tar.gz", + "sha256": "8d8971bcedb291930255359886cecb7e1cc73624385d8c574d0e239fc7f5835d", + "size": 13624 + }, { "path": "unified/dynamo_v2-0.3.4.tar.gz", "sha256": "19b6f45dc869378a86af35a4a06395cbb6fb3f8976705ded2738405a536bca25", "size": 13715 }, + { + "path": "unified/golden+pr166.patch1.tar.gz", + "sha256": "60fd970096dd0a4c78437c2e43469906c09553fb9af4f93ed3f83598e5459b6c", + "size": 392 + }, { "path": "unified/golden.tar.gz", "sha256": "fb1bda2d94f3027c96512ad5fc65ce72106ee8d0f07f78abca4f44d33edaa65b", "size": 9623 }, + { + "path": "unified/inputs+pr166.patch1.tar.gz", + "sha256": "f6722edf1539874813d50598a3a0040c7c314e7ed95295be5b4e1c56ec10a0f4", + "size": 997 + }, { "path": "unified/inputs.tar.gz", "sha256": "521faaab84e167cac39607d522504d3cd2817a21653dbd95a87de777a7305f7f", diff --git a/conformance/fixtures/unified/dynamo_v2-0.2.1.patch1.tar.gz b/conformance/fixtures/unified/dynamo_v2-0.2.1.patch1.tar.gz new file mode 100644 index 000000000..262c3b7fa --- /dev/null +++ b/conformance/fixtures/unified/dynamo_v2-0.2.1.patch1.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df96d32937bac1a127fecfe240e9a64536499c39a0fa96dfa137b7ef83777dd8 +size 3058 diff --git a/conformance/fixtures/unified/dynamo_v2-0.3.0+pr191.patch1.tar.gz b/conformance/fixtures/unified/dynamo_v2-0.3.0+pr191.patch1.tar.gz new file mode 100644 index 000000000..bbdafb3b4 --- /dev/null +++ b/conformance/fixtures/unified/dynamo_v2-0.3.0+pr191.patch1.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf5f15b4f62d08928209759677112a69a2c9151d460aa796a6006fcae978e072 +size 3077 diff --git a/conformance/fixtures/unified/dynamo_v2-0.3.2.patch1.tar.gz b/conformance/fixtures/unified/dynamo_v2-0.3.2.patch1.tar.gz new file mode 100644 index 000000000..290e2614e --- /dev/null +++ b/conformance/fixtures/unified/dynamo_v2-0.3.2.patch1.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cde85bfb976f6d75c112d4289803c72b729e6c26d12fbf88bf297cf218fdcec6 +size 608 diff --git a/conformance/fixtures/unified/dynamo_v2-0.3.3+pr166.tar.gz b/conformance/fixtures/unified/dynamo_v2-0.3.3+pr166.tar.gz new file mode 100644 index 000000000..7fdad3487 --- /dev/null +++ b/conformance/fixtures/unified/dynamo_v2-0.3.3+pr166.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb080ce2244ec6708bbe5c94171777066dd58de234511c7f540adf82906a1e0f +size 12375 diff --git a/conformance/fixtures/unified/dynamo_v2-0.3.4+pr166.patch1.tar.gz b/conformance/fixtures/unified/dynamo_v2-0.3.4+pr166.patch1.tar.gz new file mode 100644 index 000000000..567365040 --- /dev/null +++ b/conformance/fixtures/unified/dynamo_v2-0.3.4+pr166.patch1.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ef7bbd599d848112002c934105386c58e53ba19d8692bc85e9033c639f63f56 +size 486 diff --git a/conformance/fixtures/unified/dynamo_v2-0.3.4+pr166.tar.gz b/conformance/fixtures/unified/dynamo_v2-0.3.4+pr166.tar.gz new file mode 100644 index 000000000..70f367d49 --- /dev/null +++ b/conformance/fixtures/unified/dynamo_v2-0.3.4+pr166.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d8971bcedb291930255359886cecb7e1cc73624385d8c574d0e239fc7f5835d +size 13624 diff --git a/conformance/fixtures/unified/golden+pr166.patch1.tar.gz b/conformance/fixtures/unified/golden+pr166.patch1.tar.gz new file mode 100644 index 000000000..67838184a --- /dev/null +++ b/conformance/fixtures/unified/golden+pr166.patch1.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60fd970096dd0a4c78437c2e43469906c09553fb9af4f93ed3f83598e5459b6c +size 392 diff --git a/conformance/fixtures/unified/inputs+pr166.patch1.tar.gz b/conformance/fixtures/unified/inputs+pr166.patch1.tar.gz new file mode 100644 index 000000000..75fce756f --- /dev/null +++ b/conformance/fixtures/unified/inputs+pr166.patch1.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6722edf1539874813d50598a3a0040c7c314e7ed95295be5b4e1c56ec10a0f4 +size 997 diff --git a/conformance/fixtures/unified/sglang_python-0.5.16.patch1.tar.gz b/conformance/fixtures/unified/sglang_python-0.5.16.patch1.tar.gz new file mode 100644 index 000000000..a7ce7f9f7 --- /dev/null +++ b/conformance/fixtures/unified/sglang_python-0.5.16.patch1.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64f7f40efa8b44321707097e5b7c2070b6d4c6a1f39d081e3a788729c275bcbe +size 930 diff --git a/conformance/tests/common/mod.rs b/conformance/tests/common/mod.rs index a265097c0..a44c779e9 100644 --- a/conformance/tests/common/mod.rs +++ b/conformance/tests/common/mod.rs @@ -236,48 +236,45 @@ pub fn fixture_name(path: &Path) -> String { /// fixtures-batch-v1, `dynamo_v2-` under fixtures-stream-v2), ASCENDING by /// numeric version. Multiple dirs per impl are capture HISTORY (never deleted); /// readers fold them ascending so the latest capture wins per case. +pub type VersionCaptureSortKey = (Vec, bool, String); + pub fn version_dirs_ascending(root: &Path, prefix: &str) -> Vec { - let mut dirs: Vec<(Vec, PathBuf)> = std::fs::read_dir(root) + let mut dirs: Vec<(VersionCaptureSortKey, PathBuf)> = std::fs::read_dir(root) .into_iter() .flatten() .flatten() .map(|e| e.path()) - .filter(|p| { - p.is_dir() - && p.file_name().and_then(|n| n.to_str()).is_some_and(|n| { - // `.patchN` dirs are DISPLAY-ONLY overlays: an OLD parser - // binary re-run to backfill a newer case onto version `` - // (e.g. dynamo_v2-0.1.11.patch1 = the 0.1.11 binary on streamv2.5.h, - // rendered under the 0.1.11 column in HTML). They are NOT the current - // parser, so they must never join this "latest capture wins" fold — - // otherwise a stale old-binary result can shadow the real latest. - // - // `+` dirs are the same kind of thing for the same reason: - // a change-scoped capture, an older build run over the current corpus - // (see `capture_cross_version.rs`). Excluded HERE rather than at each - // call site because the version key below splits on non-digits, so - // `0.1.24+pre163` folds to [0,1,24,163] and would sort ABOVE the real - // 0.1.24 — letting a historical snapshot win "latest capture". - n.starts_with(prefix) && !n.contains(".patch") && !n.contains('+') - }) - }) - .map(|p| { - let key: Vec = p + .filter(|p| p.is_dir()) + .filter_map(|p| { + let key = p .file_name() .and_then(|n| n.to_str()) - .and_then(|n| n.strip_prefix(prefix)) - .unwrap_or("") - .split(|c: char| !c.is_ascii_digit()) - .filter(|s| !s.is_empty()) - .map(|s| s.parse().unwrap_or(0)) - .collect(); - (key, p) + .and_then(|n| version_capture_sort_key(n, prefix))?; + Some((key, p)) }) .collect(); dirs.sort(); dirs.into_iter().map(|(_, p)| p).collect() } +/// Sort a PR-qualified capture after its matching release. The PR capture represents +/// the current branch, while a plain release remains a historical comparison point. +pub fn version_capture_sort_key(name: &str, prefix: &str) -> Option { + let version = name.strip_prefix(prefix)?; + // `.patchN` dirs are display-only overlays from an old binary. They fill missing + // cases in that binary's column and must never participate in latest-capture folds. + if version.contains(".patch") { + return None; + } + let base = version.split_once('+').map_or(version, |(base, _)| base); + let numeric = base + .split(|c: char| !c.is_ascii_digit()) + .filter(|s| !s.is_empty()) + .map(|s| s.parse().unwrap_or(0)) + .collect(); + Some((numeric, version.contains('+'), version.to_string())) +} + /// One row of the `unified:` block in `conformance/utils/src/parser_families.yaml`. /// /// That block is the ONE place a family is declared for the unified tab. It replaced diff --git a/conformance/tests/unified_render.rs b/conformance/tests/unified_render.rs index cf48385ca..3a9c1d852 100644 --- a/conformance/tests/unified_render.rs +++ b/conformance/tests/unified_render.rs @@ -7,9 +7,13 @@ //! Columns: GOLDEN (authored oracle) | vLLM (captured) | Dynamo v2. //! //! The Dynamo column is a PER-FAMILY MIXTURE, the same shape as vLLM Rust's: the -//! native UnifiedParser where one exists (qwen3 today), and the v1-reasoning + -//! v2-tool split everywhere else. The split is the "before" state — it parses ALL -//! reasoning first, so reasoning interleaved with tool calls loses its position. +//! native UnifiedParser where one exists, and the v1-reasoning + v2-tool split +//! everywhere else. The split is the "before" state — it parses ALL reasoning +//! first, so reasoning interleaved with tool calls loses its position. Every +//! family in the current golden corpus (gemma4, kimi_k2, muse_glimmer, qwen3) +//! has a native UnifiedParser as of gemma4's migration, so nothing in this +//! corpus exercises the split path today; a new split-only family reintroduces +//! it. //! //! Output: `conformance/CONFORMANCE_unified.html` (standalone preview) and //! `conformance/unified/unified_results.yaml`. The exploder and packager turn that @@ -178,12 +182,9 @@ fn unified_delta_json(d: &dynamo_parsers_v2::UnifiedParserEvent) -> Value { /// Compute the Dynamo v2 unified event list for one input. /// -/// PER-FAMILY MIXTURE, the same shape as vLLM Rust's column: a family that has a -/// unified parser is parsed by it — one state machine per stream owning -/// reasoning + content + tool calls. Every other family still goes through the -/// SPLIT Dynamo serves today: the v1 reasoning parser strips reasoning over the -/// whole stream into ONE assembled field (the merge), then the v2 tool parser -/// streams the leftover content (char-by-char) preserving text/call order. +/// A family with a native unified parser is parsed by one state machine per +/// stream owning reasoning + content + tool calls. All four families in the +/// current corpus take this path; the split fallback remains for other inputs. /// /// Both paths are driven from the SAME chunking as `dynamo_chunks`, so the /// assembled row and the per-chunk rows in the popup describe one run. @@ -447,7 +448,7 @@ fn classify(family: &str, golden: &[Ev], got: &[Ev]) -> &'static str { }) .collect() }; - if cat(golden, true) == cat(got, true) && cat(golden, false) == cat(got, false) { + if gc == tc && cat(golden, true) == cat(got, true) && cat(golden, false) == cat(got, false) { return "ORDER"; } "LOSS" @@ -630,7 +631,7 @@ fn render_unified_conformance_html() { format!("
{}
", esc(todo_for(dclass))) }; let dcell = cell( - "Dynamo today (v1 reasoning + v2 tool, LIVE)", + "Dynamo today (native unified, LIVE)", &case.input, &case.golden, &events_html(&got), @@ -741,9 +742,9 @@ td.c:hover .tip,td.gold:hover .tip{{display:block}} .legend span{{display:inline-block;padding:2px 8px;border-radius:4px;margin-right:8px;font-size:12px}}

Unified conformance — reasoning + content + tool calls, one ordered stream

-

Truth column is GOLDEN — the authored, spec-derived oracle (best-effort error recovery), not captured from any implementation. Both engines are measured against it. Dynamo today is computed LIVE this run (v1 reasoning + v2 tool, composed as Dynamo serves today; Dynamo v2 is moving to a per-family mixture). vLLM 0.25.x is the documented expectation (live capture = U1; only gemma4 uses a native unified parser, the rest are combined). Hover any cell.

+

Truth column is GOLDEN — the authored, spec-derived oracle (best-effort error recovery), not captured from any implementation. Both engines are measured against it. Dynamo today is computed LIVE this run with native unified parsers for all four current corpus families. vLLM 0.25.x is the documented expectation (live capture = U1; only gemma4 uses a native unified parser, the rest are combined). Hover any cell.

matches goldendiverges (class shown)   Dynamo red: {dynamo_red}/{total} · vLLM red: {vllm_red}/{total}

- +
caseGOLDENvLLM 0.25.x
(expected)
Dynamo today
(v1 reasoning + v2 tool, LIVE)
{rows}
caseGOLDENvLLM 0.25.x
(expected)
Dynamo today
(native unified, LIVE)

Generated by conformance/tests/unified_render.rs. Cases: conformance/unified/golden_spec/ (authored by gen_unified_golden.py). Taxonomy: conformance/utils/lib/parsers/UNIFIED_CASES.md.

@@ -763,7 +764,7 @@ td.c:hover .tip,td.gold:hover .tip{{display:block}} std::fs::create_dir_all(yaml_out.parent().unwrap()).unwrap(); let feed = serde_json::json!({ "schema": "unified-results/v1", - "note": "GOLDEN = authored oracle. dynamo = LIVE (v1 reasoning + v2 tool). vllm = documented expectation (live capture pending U1).", + "note": "GOLDEN = authored oracle. dynamo = LIVE (native unified for all four current corpus families). vllm = documented expectation (live capture pending U1).", "cases": json_cases, }); std::fs::write(&yaml_out, serde_yaml::to_string(&feed).unwrap()).unwrap(); @@ -772,11 +773,19 @@ td.c:hover .tip,td.gold:hover .tip{{display:block}} out.display() ); - // Sanity: the harness computes REAL failures, not a strawman. + // Sanity: the harness computes REAL failures, not a strawman. vLLM's + // documented-expectation column supplies that signal (`vllm_red` is large + // and asserted elsewhere via the rendered legend), so this only needs to + // pin Dynamo's own invariant. assert!(total >= 14, "expected the seed corpus"); - assert!( - dynamo_red >= 3, - "expected the split to fail the interleaving cases (got {dynamo_red})" + // Every family in the current golden corpus has a native UnifiedParser + // (see the module doc), so Dynamo must match GOLDEN on every case — the + // split's interleaving-order loss no longer applies to anything here. A + // regression back to >0 means either a native parser broke, or a new + // split-only family entered the corpus without a native parser of its own. + assert_eq!( + dynamo_red, 0, + "expected every case to match golden now that every family is native (got {dynamo_red} red)" ); } @@ -841,25 +850,29 @@ fn committed_dynamo_capture_matches_the_live_parsers() { ); } // THIS build's capture: the newest version dir, via the shared helper so the - // "which capture is current" rule lives in ONE place. The helper already drops - // `.patchN` overlays and `+tag` change-scoped captures, both of which are older - // parsers and must never be mistaken for the live one. Resolving by readdir order - // instead made this guard nondeterministic — the same commit could pass against one - // shard and report parser drift against another purely on directory listing order. + // "which capture is current" rule lives in ONE place. The helper drops `.patchN` + // overlays, orders a `+tag` immediately before its matching plain release, and still + // lets a newer PR-qualified capture outrank every older release. Resolving by + // readdir order instead made this guard nondeterministic — the same commit could + // pass against one shard and report parser drift against another. let capture_dir = common::version_dirs_ascending(&root, "dynamo_v2-") .pop() .expect("no committed dynamo_v2- capture dir"); - // key -> (family, scenario, input, init), from the inputs shard. + // key -> (family, scenario, input, init), from the base inputs shard plus + // PR-qualified sparse overlays. New cases must carry their input metadata in + // the same overlay as the capture, rather than making the released shard mutable. let mut meta: BTreeMap<(String, String), (String, String, Init)> = BTreeMap::new(); - for entry in glob_yaml(&root.join("inputs")) { - let doc: InputDoc = serde_yaml::from_str(&std::fs::read_to_string(&entry).unwrap()) - .unwrap_or_else(|e| panic!("{}: {e}", entry.display())); - for (key, case) in doc.cases { - meta.insert( - (doc.family.clone(), key), - (case.scenario, case.input, case.init), - ); + for input_dir in shared_overlay_dirs(&root, "inputs") { + for entry in glob_yaml(&input_dir) { + let doc: InputDoc = serde_yaml::from_str(&std::fs::read_to_string(&entry).unwrap()) + .unwrap_or_else(|e| panic!("{}: {e}", entry.display())); + for (key, case) in doc.cases { + meta.insert( + (doc.family.clone(), key), + (case.scenario, case.input, case.init), + ); + } } } @@ -870,6 +883,10 @@ fn committed_dynamo_capture_matches_the_live_parsers() { .unwrap_or_else(|e| panic!("{}: {e}", entry.display())); for (key, committed) in doc.cases { let Some((scenario, input, init)) = meta.get(&(doc.family.clone(), key.clone())) else { + stale.push(format!( + "{} [{key}] has no input metadata in inputs or its PR overlays", + doc.family + )); continue; }; checked += 1; @@ -920,6 +937,68 @@ fn committed_dynamo_capture_matches_the_live_parsers() { ); } +/// Return the released shared shard followed by its PR-qualified sparse overlays. +/// The overlays are ordered by PR number and then patch number so a later patch is +/// authoritative if a branch intentionally extends its own metadata. +fn shared_overlay_dirs(root: &std::path::Path, base: &str) -> Vec { + let mut overlays: Vec<(u64, u64, PathBuf)> = std::fs::read_dir(root) + .into_iter() + .flatten() + .flatten() + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .filter_map(|path| { + let name = path.file_name()?.to_str()?; + let rest = name.strip_prefix(&format!("{base}+pr"))?; + let (pr, patch) = rest.split_once(".patch")?; + Some((pr.parse().ok()?, patch.parse().ok()?, path)) + }) + .collect(); + overlays.sort_by_key(|(pr, patch, _)| (*pr, *patch)); + + let mut dirs = vec![root.join(base)]; + dirs.extend(overlays.into_iter().map(|(_, _, path)| path)); + dirs +} + +#[test] +fn release_qualified_capture_order_preserves_current_and_released_owners() { + let prefix = "dynamo_v2-"; + let older_release = common::version_capture_sort_key("dynamo_v2-0.3.2", prefix).unwrap(); + let current_pr = common::version_capture_sort_key("dynamo_v2-0.3.3+pr166", prefix).unwrap(); + let current_release = common::version_capture_sort_key("dynamo_v2-0.3.3", prefix).unwrap(); + assert!(older_release < current_release); + assert!(current_release < current_pr); + assert!(common::version_capture_sort_key("dynamo_v2-0.3.3.patch1", prefix).is_none()); +} + +#[test] +fn shared_input_overlays_are_folded_after_the_released_shard() { + let root = std::env::temp_dir().join(format!( + "unified-render-overlay-order-{}", + std::process::id() + )); + std::fs::create_dir_all(root.join("inputs")).unwrap(); + std::fs::create_dir_all(root.join("inputs+pr166.patch10")).unwrap(); + std::fs::create_dir_all(root.join("inputs+pr166.patch2")).unwrap(); + std::fs::create_dir_all(root.join("inputs+pr167.patch1")).unwrap(); + + let names: Vec = shared_overlay_dirs(&root, "inputs") + .into_iter() + .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert_eq!( + names, + [ + "inputs", + "inputs+pr166.patch2", + "inputs+pr166.patch10", + "inputs+pr167.patch1", + ] + ); + std::fs::remove_dir_all(root).unwrap(); +} + fn glob_yaml(dir: &std::path::Path) -> Vec { let mut out = Vec::new(); for fam in std::fs::read_dir(dir).into_iter().flatten().flatten() { diff --git a/conformance/utils/lib/parsers/UNIFIED_CASES.md b/conformance/utils/lib/parsers/UNIFIED_CASES.md index 7414c6677..1b30e655d 100644 --- a/conformance/utils/lib/parsers/UNIFIED_CASES.md +++ b/conformance/utils/lib/parsers/UNIFIED_CASES.md @@ -54,11 +54,11 @@ The parser recovers everything it can and NEVER drops valid text, leaks markup, `MATCH` (green) · `ORDER` / `MERGE` / `LOSS` (the unification gap) · `LEAK` (markup in text, `↯`) · `ARG_MISMATCH` / `WHITESPACE` (version drift) · `ERROR` (engine hard-errored where the spec expects graceful output). -The Dynamo column is a per-family MIXTURE: `qwen3` and `muse_glimmer` run the native `UnifiedParser`; `gemma4` and `kimi_k2` still run the v1-reasoning + v2-tool split and carry the gap. Every remaining red Dynamo cell in this tab is a split-path cell. +The Dynamo column is a per-family mixture. All four current corpus families — `qwen3`, `gemma4`, `kimi_k2`, and `muse_glimmer` — run native `UnifiedParser` implementations. A future family without a native implementation falls back to the v1-reasoning + v2-tool split, and its cells must name that path explicitly. ## Quick reference — numbered taxonomy (`UNIFIED..` / `UNIFIED.-`) -Lettered case IDs keep the other suites' `group.sub` form (`1.a`, `2.b`, …). Numeric positions use `group-number` (`31-25`, `31-26`, …) so two adjacent numbers do not read like a decimal. The scenario slug is shown in parentheses. **Groups 1–9 mirror the tool-calling STREAM taxonomy** (`TOOLCALLING.streamv2.N`) as reasoning-free unified cases — this surface subsumes STREAM. **Group 10** is the reasoning axis (`REASONING.*`). **Group 11 is UNIQUE to unified**: reasoning↔tool ORDER that neither STREAM (no reasoning) nor REASONING (no ordered tool events) can express. **Group 12** is adversarial nesting — a marker of one channel inside another (P7). **Groups 30+ are REQUEST-SCOPED modes** — what the serving layer told the parser about this request, rather than what the model emitted; they use PAIRED TENS, `X0` for a mode's happy path and `X1` for its malformed counterpart, so a recovery case never sorts next to the baseline it contrasts with. 82 scenarios emitted for every family, plus 4 muse_glimmer-only. 332 cases in total. muse_glimmer emitted only 44 of the first 81 until the guided reader stopped assuming a reasoning marker PAIR and muse opens a thought with a dynamic `to=self<|message|>` header instead — so 37 guided scenarios were skipped for it. Having no marker pair is not the same as having no reasoning channel: the reader now asks the family where a thought begins, and muse answers with the same header resolver its native scan uses. These counts come from the generator's `CLEAN + EDGE` lists, not from the taxonomy map — the map reserves names the generator does not emit, which is how a stale denominator survived here before. +Lettered case IDs keep the other suites' `group.sub` form (`1.a`, `2.b`, …). Numeric positions use `group-number` (`31-25`, `31-26`, …) so two adjacent numbers do not read like a decimal. The scenario slug is shown in parentheses. **Groups 1–9 mirror the tool-calling STREAM taxonomy** (`TOOLCALLING.streamv2.N`) as reasoning-free unified cases — this surface subsumes STREAM. **Group 10** is the reasoning axis (`REASONING.*`). **Group 11 is UNIQUE to unified**: reasoning↔tool ORDER that neither STREAM (no reasoning) nor REASONING (no ordered tool events) can express. **Group 12** is adversarial nesting — a marker of one channel inside another (P7). **Groups 30+ are REQUEST-SCOPED modes** — what the serving layer told the parser about this request, rather than what the model emitted; they use PAIRED TENS, `X0` for a mode's happy path and `X1` for its malformed counterpart, so a recovery case never sorts next to the baseline it contrasts with. 82 shared scenarios are emitted for every family, plus 2 Gemma-only `call:` boundary scenarios and 4 muse_glimmer-only header scenarios, for 334 cases in total. muse_glimmer emitted only 44 of the first 81 until the guided reader stopped assuming a reasoning marker PAIR and muse opens a thought with a dynamic `to=self<|message|>` header instead — so 37 guided scenarios were skipped for it. Having no marker pair is not the same as having no reasoning channel: the reader now asks the family where a thought begins, and muse answers with the same header resolver its native scan uses. These counts come from the generator's `CLEAN + EDGE` lists, not from the taxonomy map — the map reserves names the generator does not emit, which is how a stale denominator survived here before. ### Group 1 — TC Single call - **`1.a`** (`tool_only`) One tool call, no reasoning, no surrounding text. The tool suite's baseline. @@ -267,6 +267,7 @@ Groups 1–12 vary the model OUTPUT. Groups 30+ vary the request: the resolved ` - **`31.x`** (`guided_json_unterminated_reasoning_then_wrapped_payload`) A thought whose closer never arrives, running straight into native tool markup wrapping the guided payload. `31.g` pins a wrapper around the payload OUTSIDE reasoning and `41.*` pins an unterminated thought on its own; neither asks what happens when the two meet, and that crossing is where both native families emitted the payload as REASONING and dispatched nothing. The client sees a plausible answer and never learns a call was lost. Contrast with `31.h`, where the same markup has PROSE behind it and is narration — what separates them is whether the guided payload follows, not which marker appeared. - **`31-25`** (`guided_json_quoted_bare_header_in_answer`) and **`31-26`** (`guided_json_quoted_bare_tool_header_in_answer`) A bare header the model QUOTES inside its visible answer, after the turn has already been routed. The `to=…` words are prose and only the marker is structural. Promoting the quote split one answer into an answer plus a THOUGHT for a `to=self` quote, and silently DELETED the words for a tool recipient — two different corruptions of the same class. **muse_glimmer only, by design:** the scenario needs a header whose resolution depends on turn state, and a marker-pair family has no equivalent — its opener is a fixed string meaning the same thing wherever it appears, and a duplicate opener inside a span is already `41.c`. Scope is DECLARED, never inferred from a gap: these two wrap their per-family map in `OnlyFamilies({...})`, and a family missing from a PLAIN map still fails generation. Letting any absence read as "not applicable" would hide real missing coverage behind the same cell the table uses for a genuine structural gap, and a reviewer cannot tell those apart — `test_an_undeclared_missing_family_fails_generation` pins it. - **`31-27`** (`guided_json_quoted_bare_header_after_payload`) and **`31-28`** (`guided_json_bare_tool_header_recovers_inside_a_thought`) The SCOPE pair. Turn position and open channel are independent axes, and `31-25`/`31-26` only cross one of them. `31-27` routes the turn with a PAYLOAD rather than a header, so a reader tracking only "has a header been seen" stays permissive and promotes a quote. `31-28` is the mirror: the same bare shape INSIDE an open thought is the missing-terminator recovery boundary, and a reader that closes its latch on the turn's first header demotes it and leaks `to=NAME` into the reasoning. Its golden keeps the separator SPACE in the thought (`"thinking "`), byte-for-byte what the native scan emits — a bare header absorbs that space when it opens a channel, which is right there and wrong at a recovery cut. What decides is scope, not whether a header has been seen — which is why one boolean gave the wrong answer in both directions at once. +- **`31-29`** (`gemma4_guided_json_visible_call_prose_before_reasoning`) and **`31-30`** (`gemma4_guided_json_malformed_call_prefix_before_reasoning`) are Gemma-only `call:` boundary cases. The first keeps ordinary visible `call:` prose before a thought and guided JSON. The second keeps the malformed but incomplete `call:get_weather` candidate as visible text because it lacks the `{` that starts a Gemma invoke body; the following thought and payload still route normally. They are intentionally `OnlyFamilies({gemma4})`: no other family gives this lexical prefix structural meaning, so cross-family rows would test nothing. Together they pin the boundary between plain prose and native-looking but incomplete Gemma syntax without losing the following reasoning or guided call. `31.c` and `31.d` pin **all-or-nothing**: one bad element voids the whole array and the payload goes out as text, taking the valid call with it. That is deliberate. A tool call is a side effect, so dispatching one extracted from a document that failed validation fails OPEN. Text loses nothing — the raw payload stays visible. `31.a`–`31.d` each also emit `tracing::warn!(why = "unified_guided_json_not_a_tool_call")`: the events alone are indistinguishable from a model that chose to answer in prose, so the log is the only signal the backend's guided decoding failed. diff --git a/conformance/utils/src/explode_unified_fixtures.py b/conformance/utils/src/explode_unified_fixtures.py index 6972d6b71..aadfa0323 100644 --- a/conformance/utils/src/explode_unified_fixtures.py +++ b/conformance/utils/src/explode_unified_fixtures.py @@ -21,6 +21,7 @@ Same schema shape as toolcalling/fixtures-stream-v2 (family/mode/captured_with/cases). Run: python3 conformance/utils/src/explode_unified_fixtures.py """ +import re import shutil import sys from pathlib import Path @@ -60,6 +61,29 @@ def _peer_cell(result): } +def _clear_generated_dirs(): + """Remove generated captures but retain append-only sparse patch layers.""" + for sub in ("inputs", "golden"): + shutil.rmtree(BUILD / sub, ignore_errors=True) + for d in BUILD.glob("*-*"): + if d.is_dir() and ".patch" not in d.name: + shutil.rmtree(d, ignore_errors=True) + + +def _shared_overlay_dirs(): + """Map cases in a PR Dynamo patch capture to their shared sparse overlay.""" + overlays = {} + for patch in BUILD.glob("dynamo_v2-*+pr*.patch*"): + suffix = re.search(r"(\+pr\d+\.patch\d+)$", patch.name) + if suffix is None: + continue + for fp in patch.glob("*/*.yaml"): + doc = yaml.safe_load(fp.read_text()) or {} + for key in doc.get("cases") or {}: + overlays[(fp.parent.name, key)] = suffix.group(1) + return overlays + + def main(): feed = yaml.safe_load((BUILD / "unified_results.yaml").read_text()) caps = {} @@ -77,6 +101,7 @@ def main(): "sglang_python": caps["sglang_python"].get("sglang_version") or "0.5.x", "dynamo_v2": _dynamo_v2_version(), } + shared_overlays = _shared_overlay_dirs() # A version dir is written once; accumulate cases into per-(dir, family) docs. docs = {} # (dirname, family) -> {family, mode, [model_label|captured_with], cases:{}} @@ -98,8 +123,10 @@ def slot(dirname, family, captured_with=None, model_label=None): key, fam, scenario = _case_key(cid) chunks = c.get("chunks") or [] - # inputs//.yaml — shared input (delta_text stream + metadata) - slot("inputs", fam, model_label=fam)[key] = { + # A PR patch capture owns new shared cases in a sparse overlay. The released + # shared shards remain byte-identical when this generated tree is repackaged. + shared_suffix = shared_overlays.get((fam, key), "") + slot(f"inputs{shared_suffix}", fam, model_label=fam)[key] = { "scenario": scenario, "description": c.get("description", ""), "policy": c.get("policy") or [], @@ -112,7 +139,7 @@ def slot(dirname, family, captured_with=None, model_label=None): } # golden//.yaml — the authored oracle (assembled events) - slot("golden", fam, captured_with={"golden": "v1"})[key] = { + slot(f"golden{shared_suffix}", fam, captured_with={"golden": "v1"})[key] = { "assembled": c.get("golden") or [], } @@ -132,12 +159,8 @@ def slot(dirname, family, captured_with=None, model_label=None): entry = slot(vdir, fam, captured_with={impl: ver[impl]}) entry[key] = _peer_cell(res) - # Clear old exploded dirs, then write one file per case. - for sub in ("inputs", "golden"): - shutil.rmtree(BUILD / sub, ignore_errors=True) - for d in BUILD.glob("*-*"): - if d.is_dir(): - shutil.rmtree(d, ignore_errors=True) + # Rebuild generated captures while retaining append-only sparse patch layers. + _clear_generated_dirs() n = 0 for (dirname, family), doc in docs.items(): diff --git a/conformance/utils/src/gen_unified_golden.py b/conformance/utils/src/gen_unified_golden.py index eb7667b8e..0a26c3eb1 100644 --- a/conformance/utils/src/gen_unified_golden.py +++ b/conformance/utils/src/gen_unified_golden.py @@ -814,6 +814,36 @@ def guided_surroundings(render, dynamo_note, fill=None): lambda fam: f"{control_tokens(fam)[0]}I'll use {control_tokens(fam)[2]} next{control_tokens(fam)[1]}{GUIDED_ONE_CALL}", "narrated markup stripped, thought preserved, payload survives")), + ("gemma4_guided_json_visible_call_prose_before_reasoning", + "Gemma 4 only: ordinary visible prose contains `call:` immediately before a thought and the guided JSON payload. `call:` is a lexical prefix in Gemma's native invoke grammar, but without a valid native call body here it is prose and must remain visible. This cannot be shared with another family because only Gemma's grammar assigns any possible structural meaning to this exact prefix.", + [], + [{"kind": "text", "text": "I will call: you tomorrow"}, + {"kind": "reasoning", "text": "checking"}, + {"kind": "tool_call", "name": "get_weather", "arguments": {"city": "Paris"}}], + {"starting_state": "None", "tool_output_mode": "GuidedJson", "named_tool": None}, + OnlyFamilies({ + "gemma4": ( + "I will call: you tomorrow<|channel>thought\nchecking" + GUIDED_ONE_CALL, + D("UNSUPPORTED", "vLLM base case does not use GuidedJson"), + {"verdict": "match", "note": "ordinary visible `call:` prose remains text before reasoning and the guided payload"}, + ), + })), + + ("gemma4_guided_json_malformed_call_prefix_before_reasoning", + "Gemma 4 only: an incomplete native-looking `call:get_weather` prefix sits immediately before a thought and guided JSON. It lacks the `{` that makes a Gemma invoke body, so it remains visible text rather than being silently suppressed; the following thought and guided payload still route normally. The contrast with `31-29` fixes the boundary between ordinary `call:` prose and a malformed-but-still-visible Gemma candidate.", + ["P2"], + [{"kind": "text", "text": "call:get_weather"}, + {"kind": "reasoning", "text": "secret"}, + {"kind": "tool_call", "name": "get_weather", "arguments": {"city": "Paris"}}], + {"starting_state": "None", "tool_output_mode": "GuidedJson", "named_tool": None}, + OnlyFamilies({ + "gemma4": ( + "call:get_weather<|channel>thought\nsecret" + GUIDED_ONE_CALL, + D("UNSUPPORTED", "vLLM base case does not use GuidedJson"), + {"verdict": "match", "note": "incomplete Gemma `call:get_weather` remains visible; reasoning and guided payload survive"}, + ), + })), + ("guided_json_prose_before_reasoning", "Visible prose, THEN a thought, then the payload. Every other guided case opens its thought at byte 0; when prose came first the run latched the payload buffer and the model's private thinking was surfaced to the user as the answer.", ["P2"], diff --git a/conformance/utils/src/generate_conformance_table.py b/conformance/utils/src/generate_conformance_table.py index 7d9c8a61b..657226efc 100644 --- a/conformance/utils/src/generate_conformance_table.py +++ b/conformance/utils/src/generate_conformance_table.py @@ -2197,41 +2197,69 @@ def _load_unified_fixtures(base: Path): if not (base / "inputs").is_dir(): return None - def _read_dir(name): + def _read_dir(name, include_bytes=False): out = {} # (family, case_key) -> case_doc for fp in sorted((base / name).glob("*/*.yaml")): - doc = yaml.safe_load(fp.read_text()) or {} + raw = fp.read_bytes() + doc = yaml.safe_load(raw) or {} for k, cd in (doc.get("cases") or {}).items(): - out[(fp.parent.name, k)] = cd + out[(fp.parent.name, k)] = (cd, raw) if include_bytes else cd return out - inputs = _read_dir("inputs") - golden = _read_dir("golden") + def _overlay_base(name): + return re.sub(r"\+pr\d+\.patch\d+$", "", name) + + def _merge_layers(layers, kind): + merged = {} + sources = {} + for name in layers: + for key, (record, raw) in _read_dir(name, include_bytes=True).items(): + if key in merged and raw != merged[key]: + family, case_key = key + raise ValueError( + f"conflicting shared {kind} record {family}/{case_key} in " + f"{sources[key]} and {name}; shared overlays may repeat only " + "byte-identical records" + ) + merged[key] = raw + sources.setdefault(key, name) + return {key: _read_dir(sources[key])[key] for key in merged} + + # Shared inputs and the authored golden oracle are immutable once released. New + # cases are carried by PR-qualified sparse overlays instead of rewriting them. + input_layers = ["inputs"] + sorted( + d.name + for d in base.iterdir() + if d.is_dir() and _overlay_base(d.name) == "inputs" and d.name != "inputs" + ) + golden_layers = ["golden"] + sorted( + d.name + for d in base.iterdir() + if d.is_dir() and _overlay_base(d.name) == "golden" and d.name != "golden" + ) + inputs = _merge_layers(input_layers, "input") + golden = _merge_layers(golden_layers, "golden") # impl -> [(version, dirname)] ascending. Dynamo keeps EVERY capture so the tab can # compare one parser build against another; a dict keyed by impl silently dropped # all but the last dir, which is why only one Dynamo column could ever render. # `sorted()` alone is lexicographic (0.1.9 > 0.1.10), so sort on the version key. engine_versions: dict[str, list[tuple[str, str]]] = {} for d in sorted(base.iterdir()): - if not d.is_dir() or d.name in ("inputs", "golden"): + if not d.is_dir() or _overlay_base(d.name) in ("inputs", "golden"): continue m = re.match(r"^([a-z0-9_]+)-(\d.*)$", d.name) if m: engine_versions.setdefault(m.group(1), []).append((m.group(2), d.name)) for impl in engine_versions: # `_version_sort_key` reads only the leading digits, so `0.1.24+pre163` ties with - # `0.1.24`. Break the tie so a `+tag` capture sorts BEFORE the plain release it - # qualifies: the tag marks a code state within that version (a pre-merge branch - # point), while the unadorned version is the released build. Without this the - # tagged capture wins the tie and becomes the reference, which would star the OLD - # parser and quietly measure everything against it. + # `0.1.24`. A qualified capture is a newer branch state than the release, matching + # Rust's `version_capture_sort_key`: keep it after the release so it is selected + # as the current capture while the release remains a historical comparison. engine_versions[impl].sort( - key=lambda vd: (fixtures._version_sort_key(vd[0]), "+" not in vd[0]) + key=lambda vd: (fixtures._version_sort_key(vd[0]), "+" in vd[0]) ) # The Unified tab compares every captured vLLM version. Keep each peer version # separate instead of silently replacing 0.25.1 with the newest 0.26.x shard. - engine_dirs = {impl: (vs[-1][1], vs[-1][0]) for impl, vs in engine_versions.items()} - engine_cases = {impl: _read_dir(dirname) for impl, (dirname, _v) in engine_dirs.items()} peer_by_ver = {} for impl, vers in engine_versions.items(): if impl not in ("vllm_python", "vllm_rust", "sglang_python"): @@ -2240,17 +2268,38 @@ def _read_dir(name): for ver, dirname in vers: base_ver = re.sub(r"\.patch\d+$", "", ver) target = merged.setdefault(base_ver, {}) - patch_cases = _read_dir(dirname) - overlap = set(target).intersection(patch_cases) - if overlap: - raise ValueError( - f"{dirname} rewrites existing {impl} capture cases: {sorted(overlap)!r}" - ) - target.update(patch_cases) - dynamo_by_ver = { - _base_stream_version(ver): _read_dir(dirname) - for ver, dirname in engine_versions.get("dynamo_v2", []) - } + # A `.patchN` shard is an append-only correction layer for the same + # released parser, not a second release. It may intentionally replace + # an earlier capture for a case while leaving every other base case + # intact. Applying layers in version order preserves the release's + # identity and makes the patch's listed entries authoritative. + target.update(_read_dir(dirname)) + engine_dirs = {impl: (vs[-1][1], vs[-1][0]) for impl, vs in engine_versions.items()} + engine_cases = {impl: _read_dir(dirname) for impl, (dirname, _v) in engine_dirs.items()} + for impl, captures in peer_by_ver.items(): + latest = max(captures, key=fixtures._version_sort_key) + engine_cases[impl] = captures[latest] + dynamo_by_ver = {} + for ver, dirname in engine_versions.get("dynamo_v2", []): + # `.patchN` is a sparse backfill for its released binary, but `+tag` identifies + # a distinct branch capture and must remain selectable beside that release. + display_ver = _base_stream_version(ver) + captured_cases = _read_dir(dirname) + if _PATCH_SUFFIX_RE.search(ver): + dynamo_by_ver.setdefault(display_ver, {}).update(captured_cases) + else: + dynamo_by_ver[display_ver] = captured_cases + + if dynamo_by_ver: + current_dynamo_ver, current_dynamo_cases = next(reversed(dynamo_by_ver.items())) + missing_current_cases = sorted(set(inputs).difference(current_dynamo_cases)) + if missing_current_cases: + missing = ", ".join(f"{family}/{case}" for family, case in missing_current_cases) + raise ValueError( + f"selected Dynamo v2 capture {current_dynamo_ver} lacks input case(s): {missing}; " + "add an applicable append-only .patchN overlay" + ) + engine_cases["dynamo_v2"] = current_dynamo_cases cases = [] caps = {"vllm_python": {}, "vllm_rust": {}, "sglang_python": {}} @@ -2309,6 +2358,16 @@ def _read_dir(name): } for impl in ("vllm_python", "vllm_rust", "sglang_python") }, + # A peer version with no entry for this family has no parser. A version + # that captured the family but not this case predates the case instead. + "peer_family_vers": { + impl: sorted( + ver + for ver, docs in peer_by_ver.get(impl, {}).items() + if any(captured_family == fam for captured_family, _key in docs) + ) + for impl in ("vllm_python", "vllm_rust", "sglang_python") + }, "chunks": [ {"delta_text": ic.get("delta_text", ""), "dynamo": (dyn_chunks[i].get("expected") if i < len(dyn_chunks) else []) or []} @@ -2328,10 +2387,12 @@ def _read_dir(name): "parser": edoc.get("parser"), } versions = {impl: v for impl, (_d, v) in engine_dirs.items()} + if dynamo_by_ver: + versions["dynamo_v2"] = next(reversed(dynamo_by_ver)) + for impl, captures in peer_by_ver.items(): + versions[impl] = max(captures, key=fixtures._version_sort_key) # Ascending; the tab makes the last one the reference and the rest compare-on. - versions["dynamo_v2_all"] = list( - dict.fromkeys(_base_stream_version(v) for v, _d in engine_versions.get("dynamo_v2", [])) - ) + versions["dynamo_v2_all"] = list(dynamo_by_ver) for impl in ("vllm_python", "vllm_rust", "sglang_python"): versions[f"{impl}_all"] = sorted( peer_by_ver.get(impl, {}), key=fixtures._version_sort_key @@ -2488,8 +2549,8 @@ def _cand(key, label, bucket): dyn = _assemble_stream(dyn_chunk_deltas) gsig, dsig = _sig(gold), _sig(dyn) dverd = _unified_classify(f, gold, dyn) - # Score every captured vLLM version. Missing family coverage is n/a, not an - # assumed match: Muse Glimmer has no released vLLM parser to run. + # Score every captured vLLM version. Missing family coverage and a case that + # postdates a capture are both n/a, never an assumed match. peer_results = {} for spec in peer_specs: cap = (c.get("peer_by_ver", {}).get(spec["source"], {}) @@ -2586,8 +2647,10 @@ def _cand(key, label, bucket): "version": spec["version"], "parse_mode": "unified", "leak": peer_results[spec["key"]]["verdict"] == "LEAK", "block": ({"unavailable": ( - f"{spec['label']} has no parser for {f}")} - if peer_results[spec["key"]]["events"] is None else + f"not captured at {spec['version']}; this case postdates that capture" + if spec["version"] in c["peer_family_vers"][spec["source"]] + else f"{spec['label']} has no parser for {f}")} + if peer_results[spec["key"]]["events"] is None else ({"error": peer_results[spec["key"]]["error"]} if peer_results[spec["key"]]["error"] else {"events": peer_results[spec["key"]]["events"], diff --git a/conformance/utils/src/package_fixtures.py b/conformance/utils/src/package_fixtures.py index abeb0668f..eccf43a86 100644 --- a/conformance/utils/src/package_fixtures.py +++ b/conformance/utils/src/package_fixtures.py @@ -176,7 +176,8 @@ def build_shards(tmpdir, blobs_dir, prune=False): # -/. Anything else (a stray family dir, an # overlays/ nest from a raw capture) would produce a tarball the # resolvers ignore — reject it loudly instead. - if subdir.name not in ("inputs", "golden") and not re.match(r"^[a-z0-9_]+-\d", subdir.name): + shared_overlay = re.match(r"^(inputs|golden)\+pr\d+\.patch\d+$", subdir.name) + if subdir.name not in ("inputs", "golden") and not shared_overlay and not re.match(r"^[a-z0-9_]+-\d", subdir.name): print( f" warn: skipping {tree_rel}/{subdir.name} — not inputs/ or " "-/ (normalize the capture output first)", diff --git a/conformance/utils/src/parser_families.yaml b/conformance/utils/src/parser_families.yaml index ddbe01f0a..848567517 100644 --- a/conformance/utils/src/parser_families.yaml +++ b/conformance/utils/src/parser_families.yaml @@ -188,7 +188,7 @@ markers: unified: gemma4: registry: gemma4 - native: false + native: true reasoning_parser: gemma4 tool_parser: gemma4 golden_spec: gemma4.yaml diff --git a/conformance/utils/src/unified_taxonomy.py b/conformance/utils/src/unified_taxonomy.py index 3ba5c8591..de9c54c03 100644 --- a/conformance/utils/src/unified_taxonomy.py +++ b/conformance/utils/src/unified_taxonomy.py @@ -111,6 +111,8 @@ "guided_json_quoted_bare_tool_header_in_answer": (31, "26"), "guided_json_quoted_bare_header_after_payload": (31, "27"), "guided_json_bare_tool_header_recovers_inside_a_thought": (31, "28"), + "gemma4_guided_json_visible_call_prose_before_reasoning": (31, "29"), + "gemma4_guided_json_malformed_call_prefix_before_reasoning": (31, "30"), # Group 40 — Prefilled reasoning, happy "prefilled_reasoning_with_tool": (40, "a"), "prefilled_reasoning_with_guided_json": (40, "b"), diff --git a/conformance/utils/tests/test_explode_unified_fixtures.py b/conformance/utils/tests/test_explode_unified_fixtures.py index f5b8e8663..24a407e37 100644 --- a/conformance/utils/tests/test_explode_unified_fixtures.py +++ b/conformance/utils/tests/test_explode_unified_fixtures.py @@ -10,7 +10,7 @@ if str(SRC) not in sys.path: sys.path.insert(0, str(SRC)) -from explode_unified_fixtures import _peer_cell # noqa: E402 +import explode_unified_fixtures as explode # noqa: E402 def test_peer_error_wins_over_partial_output() -> None: @@ -20,4 +20,33 @@ def test_peer_error_wins_over_partial_output() -> None: "chunks": [[{"kind": "reasoning", "text": "partial"}]], } - assert _peer_cell(result) == {"error": "UnifiedParserError::ParsingFailed"} + assert explode._peer_cell(result) == {"error": "UnifiedParserError::ParsingFailed"} + + +def test_regeneration_retains_sparse_patch_layers(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(explode, "BUILD", tmp_path) + (tmp_path / "inputs").mkdir() + (tmp_path / "golden").mkdir() + (tmp_path / "vllm_python-0.25.1").mkdir() + patch = tmp_path / "vllm_python-0.25.1.patch1" + patch.mkdir() + + explode._clear_generated_dirs() + + assert not (tmp_path / "inputs").exists() + assert not (tmp_path / "golden").exists() + assert not (tmp_path / "vllm_python-0.25.1").exists() + assert patch.is_dir() + + +def test_pr_dynamo_patch_selects_matching_shared_overlay(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(explode, "BUILD", tmp_path) + patch = tmp_path / "dynamo_v2-0.3.4+pr166.patch1" / "gemma4" + patch.mkdir(parents=True) + (patch / "UNIFIED.31-29.yaml").write_text( + "family: gemma4\ncases:\n UNIFIED.31-29: {}\n" + ) + + assert explode._shared_overlay_dirs() == { + ("gemma4", "UNIFIED.31-29"): "+pr166.patch1" + } diff --git a/conformance/utils/tests/test_model.py b/conformance/utils/tests/test_model.py index cb7129493..e6ab69a39 100644 --- a/conformance/utils/tests/test_model.py +++ b/conformance/utils/tests/test_model.py @@ -274,8 +274,27 @@ def test_unified_tab_keeps_every_captured_vllm_parser_version(model_v2): assert native["block"]["unavailable"] == "vLLM Rust 0.26.0 (stream, Combined & Unified) has no parser for muse_glimmer" -def test_unified_tab_marks_only_unsupported_vllm_families_na(model_v2): - """Every captured vLLM parser family has data for every Unified case.""" +def test_unified_default_dynamo_uses_pr_capture_and_keeps_release_history(tmp_path): + """The actual packaged 0.3.4 fixtures retain the release and select the PR capture.""" + page_path = tmp_path / "CONFORMANCE_v2.html" + subprocess.run( + [str(UTILS / "render_table_v2.sh"), "--output", str(page_path)], + check=True, + capture_output=True, + cwd=REPO, + ) + tab = _tab(_read_model(page_path, "render_table_v2.sh"), "tab-unified") + dynamo = next(candidate for candidate in tab["candidates"] if candidate["key"] == "dynamo") + release = next(candidate for candidate in tab["candidates"] if candidate["key"] == "dynamo@0.3.4") + + assert dynamo["label"] == "Dynamo v2 Rust 0.3.4+pr166 (stream, Combined & Unified)" + assert dynamo["default_bucket"] == "A" + assert release["label"] == "Dynamo v2 Rust 0.3.4 (stream, Combined & Unified)" + assert release["default_bucket"] == "C" + + +def test_unified_tab_marks_unsupported_and_postdated_vllm_cases_na(model_v2): + """n/a distinguishes an unsupported family from a case absent from an old capture.""" tab = _tab(model_v2, "tab-unified") peer_keys = {candidate["key"] for candidate in tab["candidates"] if candidate["impl"] == "vllm"} assert peer_keys == {"vllm", "vllm_python@0.26.0", "vllm_rust", "vllm_rust@0.26.0"} @@ -284,9 +303,26 @@ def test_unified_tab_marks_only_unsupported_vllm_families_na(model_v2): unavailable = [cell["cmp"][key].get("na") == 1 for cell in row["cells"].values()] if row["family"] == "muse_glimmer": assert all(unavailable), f"{key} must say n/a for Muse" - else: + elif row["family"] != "gemma4": assert not any(unavailable), f"{key} is missing captured cases for {row['family']}" + gemma = next(row for row in tab["rows"] if row["family"] == "gemma4") + for key in peer_keys: + for cell in gemma["cells"].values(): + if cell["cmp"][key].get("na") == 1: + peer = next(candidate for candidate in cell["tooltip"]["candidates"] if candidate["key"] == key) + assert "this case postdates that capture" in peer["block"]["unavailable"] + + for scenario in ( + "gemma4_guided_json_visible_call_prose_before_reasoning", + "gemma4_guided_json_malformed_call_prefix_before_reasoning", + ): + for key in peer_keys: + cell = gemma["cells"][scenario] + assert cell["cmp"][key].get("na") == 1 + peer = next(candidate for candidate in cell["tooltip"]["candidates"] if candidate["key"] == key) + assert "this case postdates that capture" in peer["block"]["unavailable"] + _IMPL_KEYS = ("dynamo_v1", "dynamo_v2", "vllm_rust", "vllm_python", "sglang_python") diff --git a/conformance/utils/tests/test_unified_fixture_overlays.py b/conformance/utils/tests/test_unified_fixture_overlays.py new file mode 100644 index 000000000..c47fc7e11 --- /dev/null +++ b/conformance/utils/tests/test_unified_fixture_overlays.py @@ -0,0 +1,363 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +import sys + +import pytest +import yaml + +UTILS_SRC = Path(__file__).resolve().parents[1] / "src" +if str(UTILS_SRC) not in sys.path: + sys.path.insert(0, str(UTILS_SRC)) + +import generate_conformance_table as table # noqa: E402 + + +def _write_case(root, dirname, family, key, body, **metadata): + path = root / dirname / family / f"{key}.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + yaml.safe_dump( + {"family": family, "mode": "unified", **metadata, "cases": {key: body}}, + sort_keys=False, + ) + ) + + +def test_sparse_unified_patch_merges_with_base_and_overrides_in_order(tmp_path): + family = "gemma4" + base_key = "UNIFIED.1.a" + patch_key = "UNIFIED.1.b" + for key, scenario in ( + (base_key, "base_case"), + (patch_key, "patch_case"), + ): + _write_case( + tmp_path, + "inputs", + family, + key, + {"scenario": scenario, "chunks": [{"delta_text": scenario}]}, + model_label=family, + ) + _write_case( + tmp_path, + "golden", + family, + key, + {"assembled": [{"kind": "text", "text": scenario}]}, + captured_with={"golden": "v1"}, + ) + + _write_case( + tmp_path, + "dynamo_v2-0.2.1", + family, + base_key, + {"assembled": [{"kind": "text", "text": "base"}], "chunks": []}, + captured_with={"dynamo_v2": "0.2.1"}, + ) + _write_case( + tmp_path, + "dynamo_v2-0.2.1.patch1", + family, + base_key, + {"assembled": [{"kind": "text", "text": "patched"}], "chunks": []}, + captured_with={"dynamo_v2": "0.2.1.patch1"}, + ) + _write_case( + tmp_path, + "dynamo_v2-0.2.1.patch1", + family, + patch_key, + {"assembled": [{"kind": "text", "text": "patch-only"}], "chunks": []}, + captured_with={"dynamo_v2": "0.2.1.patch1"}, + ) + + cases, _caps, versions = table._load_unified_fixtures(tmp_path) + by_scenario = {case["scenario"]: case for case in cases} + + assert versions["dynamo_v2_all"] == ["0.2.1"] + assert by_scenario["base_case"]["dynamo_by_ver"]["0.2.1"]["assembled"][0]["text"] == "patched" + assert by_scenario["patch_case"]["dynamo_by_ver"]["0.2.1"]["assembled"][0]["text"] == "patch-only" + + +def test_pr_overlay_merges_shared_inputs_and_golden_without_rewriting_releases(tmp_path): + family = "gemma4" + base_key = "UNIFIED.1.a" + pr_key = "UNIFIED.1.b" + for dirname, key, body, metadata in ( + ("inputs", base_key, {"scenario": "released", "chunks": [{"delta_text": "base"}]}, {"model_label": family}), + ("golden", base_key, {"assembled": [{"kind": "text", "text": "base"}]}, {"captured_with": {"golden": "v1"}}), + ("inputs+pr166.patch1", pr_key, {"scenario": "pr_only", "chunks": [{"delta_text": "overlay"}]}, {"model_label": family}), + ("golden+pr166.patch1", pr_key, {"assembled": [{"kind": "text", "text": "overlay"}]}, {"captured_with": {"golden": "v1"}}), + ): + _write_case(tmp_path, dirname, family, key, body, **metadata) + for key, text in ((base_key, "base"), (pr_key, "overlay")): + _write_case( + tmp_path, + "dynamo_v2-0.3.4+pr166.patch1", + family, + key, + {"assembled": [{"kind": "text", "text": text}], "chunks": []}, + captured_with={"dynamo_v2": "0.3.4+pr166"}, + ) + + cases, _caps, _versions = table._load_unified_fixtures(tmp_path) + + assert {case["scenario"] for case in cases} == {"released", "pr_only"} + assert next(case for case in cases if case["scenario"] == "pr_only")["golden"] == [{"kind": "text", "text": "overlay"}] + + +@pytest.mark.parametrize( + ("dirname", "base_body", "overlay_body", "kind"), + [ + ( + "inputs", + {"scenario": "released", "chunks": [{"delta_text": "base"}]}, + {"scenario": "changed", "chunks": [{"delta_text": "base"}]}, + "input", + ), + ( + "golden", + {"assembled": [{"kind": "text", "text": "base"}]}, + {"assembled": [{"kind": "text", "text": "changed"}]}, + "golden", + ), + ], +) +def test_shared_overlay_rejects_conflicting_duplicate_records( + tmp_path, dirname, base_body, overlay_body, kind +): + family = "gemma4" + key = "UNIFIED.1.a" + if dirname == "golden": + _write_case( + tmp_path, + "inputs", + family, + key, + {"scenario": "released", "chunks": [{"delta_text": "base"}]}, + model_label=family, + ) + else: + _write_case( + tmp_path, + "golden", + family, + key, + {"assembled": [{"kind": "text", "text": "base"}]}, + captured_with={"golden": "v1"}, + ) + metadata = {"model_label": family} if dirname == "inputs" else {"captured_with": {"golden": "v1"}} + _write_case(tmp_path, dirname, family, key, base_body, **metadata) + _write_case(tmp_path, f"{dirname}+pr166.patch1", family, key, overlay_body, **metadata) + + with pytest.raises(ValueError, match=rf"conflicting shared {kind} record gemma4/UNIFIED\.1\.a"): + table._load_unified_fixtures(tmp_path) + + +@pytest.mark.parametrize("dirname, body", [ + ("inputs", {"scenario": "same", "chunks": [{"delta_text": "same"}]}), + ("golden", {"assembled": [{"kind": "text", "text": "same"}]}), +]) +def test_shared_overlay_accepts_byte_identical_duplicate_records(tmp_path, dirname, body): + family = "gemma4" + key = "UNIFIED.1.a" + if dirname == "golden": + _write_case( + tmp_path, + "inputs", + family, + key, + {"scenario": "same", "chunks": [{"delta_text": "same"}]}, + model_label=family, + ) + else: + _write_case( + tmp_path, + "golden", + family, + key, + {"assembled": [{"kind": "text", "text": "same"}]}, + captured_with={"golden": "v1"}, + ) + metadata = {"model_label": family} if dirname == "inputs" else {"captured_with": {"golden": "v1"}} + _write_case(tmp_path, dirname, family, key, body, **metadata) + _write_case(tmp_path, f"{dirname}+pr166.patch1", family, key, body, **metadata) + _write_case( + tmp_path, + "dynamo_v2-0.3.4+pr166", + family, + key, + {"assembled": [{"kind": "text", "text": "same"}], "chunks": []}, + captured_with={"dynamo_v2": "0.3.4+pr166"}, + ) + + cases, _caps, _versions = table._load_unified_fixtures(tmp_path) + + assert len(cases) == 1 + + +def test_shared_overlay_rejects_semantically_equal_but_byte_different_record(tmp_path): + family = "gemma4" + key = "UNIFIED.1.a" + _write_case( + tmp_path, + "inputs", + family, + key, + {"scenario": "same", "chunks": [{"delta_text": "same"}]}, + model_label=family, + ) + overlay = tmp_path / "inputs+pr166.patch1" / family / f"{key}.yaml" + overlay.parent.mkdir(parents=True) + overlay.write_text( + "family: gemma4\nmode: unified\nmodel_label: gemma4\ncases:\n" + " UNIFIED.1.a: {chunks: [{delta_text: same}], scenario: same}\n" + ) + + with pytest.raises(ValueError, match=r"conflicting shared input record gemma4/UNIFIED\.1\.a"): + table._load_unified_fixtures(tmp_path) + + +def test_unified_qualified_capture_is_current_and_keeps_the_release(tmp_path): + family = "gemma4" + key = "UNIFIED.1.a" + _write_case( + tmp_path, + "inputs", + family, + key, + {"scenario": "capture_history", "chunks": [{"delta_text": "x"}]}, + model_label=family, + ) + _write_case( + tmp_path, + "golden", + family, + key, + {"assembled": [{"kind": "text", "text": "x"}]}, + captured_with={"golden": "v1"}, + ) + for version, text in (("0.3.4", "release"), ("0.3.4+pr166", "branch")): + _write_case( + tmp_path, + f"dynamo_v2-{version}", + family, + key, + {"assembled": [{"kind": "text", "text": text}], "chunks": []}, + captured_with={"dynamo_v2": version}, + ) + + cases, _caps, versions = table._load_unified_fixtures(tmp_path) + + assert versions["dynamo_v2"] == "0.3.4+pr166" + assert versions["dynamo_v2_all"] == ["0.3.4", "0.3.4+pr166"] + assert cases[0]["dynamo_by_ver"]["0.3.4"]["assembled"][0]["text"] == "release" + assert cases[0]["dynamo_by_ver"]["0.3.4+pr166"]["assembled"][0]["text"] == "branch" + + +def test_selected_current_capture_requires_every_input_or_a_sparse_overlay(tmp_path): + family = "gemma4" + captured_key = "UNIFIED.1.a" + missing_key = "UNIFIED.1.b" + for key in (captured_key, missing_key): + _write_case( + tmp_path, + "inputs", + family, + key, + {"scenario": key, "chunks": [{"delta_text": key}]}, + model_label=family, + ) + _write_case( + tmp_path, + "golden", + family, + key, + {"assembled": [{"kind": "text", "text": key}]}, + captured_with={"golden": "v1"}, + ) + _write_case( + tmp_path, + "dynamo_v2-0.3.4+pr166", + family, + captured_key, + {"assembled": [{"kind": "text", "text": "branch"}], "chunks": []}, + captured_with={"dynamo_v2": "0.3.4+pr166"}, + ) + + with pytest.raises(ValueError, match=r"0\.3\.4\+pr166 lacks input case\(s\): gemma4/UNIFIED\.1\.b"): + table._load_unified_fixtures(tmp_path) + + _write_case( + tmp_path, + "dynamo_v2-0.3.4+pr166.patch1", + family, + missing_key, + {"assembled": [{"kind": "text", "text": "overlay"}], "chunks": []}, + captured_with={"dynamo_v2": "0.3.4+pr166.patch1"}, + ) + + cases, _caps, versions = table._load_unified_fixtures(tmp_path) + + assert versions["dynamo_v2"] == "0.3.4+pr166" + assert {case["scenario"] for case in cases} == {captured_key, missing_key} + + +def test_sparse_peer_patch_overrides_base_case_without_changing_release(tmp_path): + family = "gemma4" + base_key = "UNIFIED.1.a" + patch_key = "UNIFIED.1.b" + for key, scenario in ((base_key, "base_case"), (patch_key, "patch_case")): + _write_case( + tmp_path, + "inputs", + family, + key, + {"scenario": scenario, "chunks": [{"delta_text": scenario}]}, + model_label=family, + ) + _write_case( + tmp_path, + "golden", + family, + key, + {"assembled": [{"kind": "text", "text": scenario}]}, + captured_with={"golden": "v1"}, + ) + _write_case( + tmp_path, + "vllm_python-0.25.1", + family, + base_key, + {"assembled": [{"kind": "text", "text": "base"}], "chunks": []}, + captured_with={"vllm_python": "0.25.1"}, + ) + _write_case( + tmp_path, + "vllm_python-0.25.1.patch1", + family, + base_key, + {"assembled": [{"kind": "text", "text": "patched"}], "chunks": []}, + captured_with={"vllm_python": "0.25.1.patch1"}, + ) + _write_case( + tmp_path, + "vllm_python-0.25.1.patch1", + family, + patch_key, + {"assembled": [{"kind": "text", "text": "patch-only"}], "chunks": []}, + captured_with={"vllm_python": "0.25.1.patch1"}, + ) + + cases, caps, versions = table._load_unified_fixtures(tmp_path) + by_scenario = {case["scenario"]: case for case in cases} + + assert versions["vllm_python"] == "0.25.1" + assert versions["vllm_python_all"] == ["0.25.1"] + assert caps["vllm_python"]["UNIFIED.base_case.gemma4"]["assembled"][0]["text"] == "patched" + assert by_scenario["base_case"]["peer_by_ver"]["vllm_python"]["0.25.1"]["assembled"][0]["text"] == "patched" + assert by_scenario["patch_case"]["peer_by_ver"]["vllm_python"]["0.25.1"]["assembled"][0]["text"] == "patch-only" diff --git a/conformance/utils/tests/test_unified_taxonomy_covers_corpus.py b/conformance/utils/tests/test_unified_taxonomy_covers_corpus.py index 5d7fcc2ab..a86434c93 100644 --- a/conformance/utils/tests/test_unified_taxonomy_covers_corpus.py +++ b/conformance/utils/tests/test_unified_taxonomy_covers_corpus.py @@ -117,11 +117,16 @@ def test_case_labels_use_numeric_positions_from_y_onward() -> None: assert tax("guided_json_quoted_bare_tool_header_in_answer") == (31, "26") assert tax("guided_json_quoted_bare_header_after_payload") == (31, "27") assert tax("guided_json_bare_tool_header_recovers_inside_a_thought") == (31, "28") + assert tax("gemma4_guided_json_visible_call_prose_before_reasoning") == (31, "29") + assert tax("gemma4_guided_json_malformed_call_prefix_before_reasoning") == (31, "30") assert case_label("guided_json_quoted_bare_header_in_answer") == "31-25" assert case_label("guided_json_quoted_bare_tool_header_in_answer") == "31-26" assert case_label("guided_json_quoted_bare_header_after_payload") == "31-27" assert case_label("guided_json_bare_tool_header_recovers_inside_a_thought") == "31-28" + assert case_label("gemma4_guided_json_visible_call_prose_before_reasoning") == "31-29" + assert case_label("gemma4_guided_json_malformed_call_prefix_before_reasoning") == "31-30" assert numbered_id("guided_json_quoted_bare_header_in_answer") == "UNIFIED.31-25" + assert numbered_id("gemma4_guided_json_visible_call_prose_before_reasoning") == "UNIFIED.31-29" assert case_label("guided_json_invalid_call") == "31.a" assert numbered_id("guided_json_invalid_call") == "UNIFIED.31.a" assert not [sub for _group, sub in UNIFIED_TAX.values() if re.fullmatch(r"[a-z]{2,}", sub)] @@ -132,10 +137,12 @@ def test_case_labels_use_numeric_positions_from_y_onward() -> None: "guided_json_quoted_bare_tool_header_in_answer", "guided_json_quoted_bare_header_after_payload", "guided_json_bare_tool_header_recovers_inside_a_thought", + "gemma4_guided_json_visible_call_prose_before_reasoning", + "gemma4_guided_json_malformed_call_prefix_before_reasoning", ), key=taxonomy_sort_key, ) - assert [tax(scenario)[1] for scenario in ordered] == ["25", "26", "27", "28"] + assert [tax(scenario)[1] for scenario in ordered] == ["25", "26", "27", "28", "29", "30"] # --- End-to-end test cases: the SAME mapping is written in two places ----------- @@ -474,12 +481,17 @@ def test_unified_case_counts_match_the_generator(): """ per_family = {fam: len(build_cases(fam)) for fam in FAMILIES} shared = min(per_family.values()) + gemma_only = { + "gemma4_guided_json_visible_call_prose_before_reasoning", + "gemma4_guided_json_malformed_call_prefix_before_reasoning", + } assert per_family["muse_glimmer"] > shared, ( "muse_glimmer should carry the OnlyFamilies scenarios on top of the shared set" ) for fam in FAMILIES: - if fam != "muse_glimmer": + if fam not in {"gemma4", "muse_glimmer"}: assert per_family[fam] == shared, f"{fam} diverged from the shared set" + assert per_family["gemma4"] == shared + len(gemma_only) assert sum(per_family.values()) == sum(len(build_cases(f)) for f in FAMILIES) @@ -535,7 +547,7 @@ def _packed_layer(shard: str, fam: str, field_map): def _taxonomy_scenarios(fam: str): - """Taxonomy id (`UNIFIED.31-28`) -> scenario slug, from the packaged INPUTS shard. + """Taxonomy id (`UNIFIED.31-28`) -> scenario slug, from packaged input layers. The golden shard keys by taxonomy id and carries no scenario field, so the mapping comes from the one shard holding both. A key JOIN, not a value normalization. @@ -548,15 +560,16 @@ def _taxonomy_scenarios(fam: str): if cached is not None: return cached out = {} - path = UTILS.parent / "fixtures" / "unified" / "inputs.tar.gz" - with tarfile.open(path) as tar: - for member in tar.getmembers(): - if not member.name.endswith(".yaml") or f"/{fam}/" not in member.name: - continue - doc = yaml.safe_load(tar.extractfile(member).read()) or {} - for cid, case in (doc.get("cases") or {}).items(): - if case.get("scenario"): - out[cid] = case["scenario"] + fixtures = UTILS.parent / "fixtures" / "unified" + for path in sorted(fixtures.glob("inputs*.tar.gz")): + with tarfile.open(path) as tar: + for member in tar.getmembers(): + if not member.name.endswith(".yaml") or f"/{fam}/" not in member.name: + continue + doc = yaml.safe_load(tar.extractfile(member).read()) or {} + for cid, case in (doc.get("cases") or {}).items(): + if case.get("scenario"): + out[cid] = case["scenario"] _taxonomy_scenarios._cache[fam] = out return out @@ -586,8 +599,12 @@ def test_every_case_triple_is_identical_at_every_layer(): checked = 0 for fam in FAMILIES: spec = _emitted_spec(fam) - packed_in = _packed_layer("inputs.tar.gz", fam, {"input": "input", "init": "init"}) - packed_gold = _packed_layer("golden.tar.gz", fam, {"golden": "assembled"}) + packed_in = {} + packed_gold = {} + for path in sorted((UTILS.parent / "fixtures" / "unified").glob("inputs*.tar.gz")): + packed_in.update(_packed_layer(path.name, fam, {"input": "input", "init": "init"})) + for path in sorted((UTILS.parent / "fixtures" / "unified").glob("golden*.tar.gz")): + packed_gold.update(_packed_layer(path.name, fam, {"golden": "assembled"})) for cid, case in build_cases(fam).items(): scenario = _scenario_of(cid, fam) diff --git a/parsers/v2/Cargo.toml b/parsers/v2/Cargo.toml index a7cd6d31b..4b3649496 100644 --- a/parsers/v2/Cargo.toml +++ b/parsers/v2/Cargo.toml @@ -7,6 +7,9 @@ edition.workspace = true license = "Apache-2.0" description = "Token-incremental tool-call streaming parsers for Dynamo" +[features] +test-utils = [] + [dependencies] openai-harmony = { workspace = true } anyhow = { workspace = true } diff --git a/parsers/v2/README.md b/parsers/v2/README.md index e9937f3b9..adb8ed9e7 100644 --- a/parsers/v2/README.md +++ b/parsers/v2/README.md @@ -53,7 +53,7 @@ Reasoning families: | **Gemma 4 channel** | `<\|channel>thought\n...` with role-label prefix stripped | `gemma4_parser.rs` | Google Gemma 4 thinking models | | **ATEM channel** | `<\|start\|>assistant to=self<\|message\|>...<\|eom\|>` — the recipient picks the channel, so there is no delimiter pair | `muse_glimmer_parser.rs` | Muse-Glimmer-30B | -Streaming (v2) implementations exist today for `harmony`, `deepseek_v4` (DSML), `qwen3_coder`, and `muse_glimmer` (ATEM); the remaining families run on the v1 batch parser until their streaming port lands. +Streaming (v2) implementations exist today for `harmony`, `deepseek_v4` (DSML), `qwen3_coder`, `gemma4`, and `muse_glimmer` (ATEM); the remaining families run on the v1 batch parser until their streaming port lands. ## Why It Mimics vLLM Rust diff --git a/parsers/v2/src/tool_calling/gemma4.rs b/parsers/v2/src/tool_calling/gemma4.rs index 55d7f124a..a4950b4f0 100644 --- a/parsers/v2/src/tool_calling/gemma4.rs +++ b/parsers/v2/src/tool_calling/gemma4.rs @@ -7,21 +7,25 @@ //! `<|tool_call>call:NAME{key:<|"|>value<|"|>, key2:42, key3:[...]}` //! with bare unquoted keys, `<|"|>`-delimited strings, nested objects/arrays, and //! MULTIPLE calls concatenated with NO separator. START = `<|tool_call>`, -//! END = `` (asymmetric). +//! END = `` (asymmetric), and the block IS the invoke — the same +//! `` closes both. //! -//! The streaming concern (buffering, chunk-split marker safety, normal_text -//! suppression, EOF-truncation drop) is owned here. Per-block name + value typing -//! is delegated to the v1 batch parser `try_tool_call_parse_gemma4`, so a streamed -//! call matches exactly what the batch parser produces. A call is emitted only -//! once its complete `<|tool_call> ... ` block has streamed; an -//! incomplete trailing block at EOF is DROPPED (v1 parity), not emitted with empty -//! arguments. +//! The streaming concerns (buffering, chunk-split marker safety, normal_text +//! suppression, orphan-close stripping, EOF-truncation drop) live in the shared +//! [`WrappedBlockScanner`], not here. Gemma 4 supplies only what its markers +//! cannot express, through [`InvokeScan`]: `` and `call:` both occur +//! legitimately INSIDE a `<|"|>`-delimited string value, so where an invoke ends +//! and whether a `call:` opens one are answered by the v1 balanced, +//! string-aware scan rather than by `find`. One scanner, one boundary +//! definition, shared with the batch parser and with the unified parser +//! ([`crate::unified::gemma4`]). //! -//! Block-completion detection (where the `` end marker actually closes -//! a call, ignoring a bare `` literal embedded inside a `<|"|>` string -//! value) reuses the v1 helper `find_tool_call_end_position_gemma4` rather than -//! re-implementing the balanced-brace + string-delimiter scan, so the streaming -//! and batch paths share one block boundary definition. +//! Per-invoke name + value typing is delegated to the v1 batch parser +//! `parse_one_tool_call_gemma4`, so a streamed call matches exactly what the +//! batch parser produces. A call is emitted only once its complete block has +//! streamed; at EOF a body that balanced without its close marker is still +//! recovered (v1 parity, case `5.b`), while one truncated mid-value is DROPPED +//! rather than emitted with empty arguments. //! //! Arguments are re-serialized in source key order because the v1 parser builds //! them from a `serde_json::Map` (a `BTreeMap` without the `preserve_order` @@ -29,277 +33,312 @@ //! exact JSON string in the model-emitted order (the order vLLM's Rust parser also //! preserves), so order has to be pinned to source order. -use crate::tool_calling::scan::reorder_arguments; +use crate::tool_calling::scan::{ + BareRecoveryLatch, InvokeBoundary, InvokeBoundaryFactory, InvokeEmitter, InvokeLatch, + WrappedBlockScanner, WrappedBlockSpec, reorder_arguments, +}; use crate::tool_calling::v1core::ToolDefinition; use crate::tool_calling::v1core::gemma4::{ - detect_tool_call_start_gemma4, find_tool_call_end_position_gemma4, try_tool_call_parse_gemma4, + has_bare_call_body_start_gemma4, is_call_prefix_boundary, parse_one_tool_call_gemma4, }; use crate::tool_calling::traits::{Tool, ToolCallDelta, ToolParseResult, ToolParser}; -const TOOL_CALL_START: &str = "<|tool_call>"; +pub(crate) const TOOL_CALL_START: &str = "<|tool_call>"; +pub(crate) const TOOL_CALL_END: &str = ""; const CALL_PREFIX: &str = "call:"; const STRING_DELIM: &str = "<|\"|>"; -/// Which kind of opener started the call span currently being buffered. -#[derive(Clone, Copy, PartialEq, Eq)] -enum Block { - /// A wrapped call opened by a literal `<|tool_call>` marker (still in the - /// buffer as the block's leading marker). - Wrapped, - /// A bare call opened by a boundary `call:` token with NO `<|tool_call>` - /// opener (the "orphan close" / "missing start" recovery shape). The v1 batch - /// parser recovers these; the streaming parser does too, so the call body is - /// recovered as a tool call instead of leaking as normal_text. Genuine prose - /// before the `call:` token has already been emitted as normal_text before - /// the block was entered. - Bare, +#[cfg(any(test, feature = "test-utils"))] +std::thread_local! { + static BOUNDARY_EXAMINED_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; } -/// Stream parser for Gemma 4 tool calls. -pub struct Gemma4ToolStreamParser { - buffer: String, - /// `Some(kind)` once a call opener (`<|tool_call>` or a recoverable bare - /// `call:`) has been entered and we are buffering the call body until its - /// complete `` end marker arrives. While set, intra-call bytes - /// never leak to `normal_text`; suppression is scoped to this in-block window - /// (narration after a call is preserved, matching the engines). - block: Option, - /// Stable parser-local tool index, incremented per emitted call. - next_index: usize, - tools: Vec, +#[cfg(any(test, feature = "test-utils"))] +#[doc(hidden)] +pub fn reset_boundary_examined_bytes() { + BOUNDARY_EXAMINED_BYTES.with(|examined| examined.set(0)); } -impl Gemma4ToolStreamParser { - pub fn new(tools: &[Tool]) -> Self { - Self { - buffer: String::new(), - block: None, - next_index: 0, - tools: tools.iter().map(ToolDefinition::from).collect(), - } - } +#[cfg(any(test, feature = "test-utils"))] +#[doc(hidden)] +pub fn boundary_examined_bytes() -> usize { + BOUNDARY_EXAMINED_BYTES.with(std::cell::Cell::get) +} - fn drain(&mut self, flush: bool) -> anyhow::Result { - let mut out = ToolParseResult::default(); +fn count_boundary_bytes(bytes: usize) { + #[cfg(any(test, feature = "test-utils"))] + BOUNDARY_EXAMINED_BYTES.with(|examined| examined.set(examined.get() + bytes)); + #[cfg(not(any(test, feature = "test-utils")))] + let _ = bytes; +} + +#[derive(Default)] +struct Gemma4InvokeProgress { + cursor: usize, + name_started: bool, + depth: usize, + in_string: bool, + body_end: Option, + invalid: bool, +} - loop { - if let Some(kind) = self.block { - // Inside a call span. For a wrapped block the leading `<|tool_call>` - // is still in the buffer; for a bare block the buffer starts at the - // `call:` token. Find the end of the FIRST complete `}` - // block (the v1 helper respects brace balance + `<|"|>` strings, so - // an embedded `` literal inside a string value does not - // close the block early). The helper returns the position after the - // LAST complete block, so isolating the buffer to a single leading - // block (next opener onward held back) keeps us emitting one call - // per iteration with the correct per-call source order. - let next_opener = self.next_opener_after_current(kind); - let scan = match next_opener { - Some(s) => &self.buffer[..s], - None => &self.buffer[..], - }; - match find_tool_call_end_position_gemma4(scan) { - Some(end) => { - let block = self.buffer[..end].to_string(); - self.buffer.drain(..end); - self.block = None; - self.emit_block(&block, kind, &mut out)?; - continue; +impl Gemma4InvokeProgress { + fn reset(&mut self) { + *self = Self::default(); + } + + fn end(&mut self, text: &str, flush: bool) -> Option { + if self.invalid || !text.starts_with(CALL_PREFIX) { + self.invalid = true; + return None; + } + if self.body_end.is_none() { + self.cursor = self.cursor.max(CALL_PREFIX.len()); + while self.cursor < text.len() { + let rest = &text[self.cursor..]; + let ch = rest.chars().next()?; + count_boundary_bytes(ch.len_utf8()); + if self.depth == 0 { + if ch == '{' && self.name_started { + self.depth = 1; + } else if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.') { + self.name_started = true; + } else { + self.invalid = true; + return None; } - None => { - // No complete block in the leading segment. If a later - // opener exists, the leading block is malformed/incomplete - // (e.g. missing end before the next call); drop it and - // continue scanning from the next opener. - if let Some(s) = next_opener { - tracing::warn!( - why = "gemma4_incomplete_tool_call", - next_index = self.next_index, - "Gemma 4 stream dropped an incomplete leading block before the next call" - ); - self.buffer.drain(..s); - self.block = None; // re-enter on the next opener below - } else if flush { - // EOF with no `` end marker. The streamv2 - // conformance tab grades dynamo's stream output against - // dynamo's own v1 BATCH output, and the v1 batch parser - // recovers a call whose body is complete even when the end - // marker never streamed (batch 5.a/5.d). Match it: - // delegate the recover-or-drop decision to the v1 parser - // via emit_block. A complete body is recovered; a - // genuinely incomplete body (truncated mid-value, 5.c) - // yields no call and is dropped (emit_block logs it). The - // buffered markup never leaks to normal_text either way. - let block = std::mem::take(&mut self.buffer); - self.block = None; - self.emit_block(&block, kind, &mut out)?; - break; - } else { - break; + self.cursor += ch.len_utf8(); + continue; + } + if STRING_DELIM.starts_with(rest) && rest.len() < STRING_DELIM.len() { + return None; + } + if rest.starts_with(STRING_DELIM) { + self.in_string = !self.in_string; + self.cursor += STRING_DELIM.len(); + continue; + } + if !self.in_string { + match ch { + '{' => self.depth += 1, + '}' => { + let Some(depth) = self.depth.checked_sub(1) else { + self.invalid = true; + return None; + }; + self.depth = depth; + if self.depth == 0 { + self.body_end = Some(self.cursor); + self.cursor += ch.len_utf8(); + break; + } } - // Fall through to the not-in-block opener scan. + _ => {} } } + self.cursor += ch.len_utf8(); + } + } + let body_end = self.body_end?; + let after = &text[body_end + 1..]; + if after.starts_with(TOOL_CALL_END) { + let mut end = body_end + 1 + TOOL_CALL_END.len(); + while text[end..].starts_with(TOOL_CALL_END) { + end += TOOL_CALL_END.len(); } + return Some(end); + } + if flush && after.trim().is_empty() { + return Some(body_end + 1); + } + None + } +} - // Not in a block: find the next opener — either a literal `<|tool_call>` - // marker, or a bare boundary `call:` token that is a recoverable call - // (orphan-close / missing-start shape). Whichever comes first wins. - let wrapped_pos = self.buffer.find(TOOL_CALL_START); - let bare_pos = self.first_bare_call_opener(); - let opener = match (wrapped_pos, bare_pos) { - (Some(w), Some(b)) if w <= b => Some((w, Block::Wrapped)), - (Some(_), Some(b)) => Some((b, Block::Bare)), - (Some(w), None) => Some((w, Block::Wrapped)), - (None, Some(b)) => Some((b, Block::Bare)), - (None, None) => None, - }; - - match opener { - Some((start, kind)) => { - // Text before the opener is user-visible normal_text. For the - // bare case this is the genuine prose prefix (e.g. 5.g's - // "I will check that. "), which must stay normal_text. - if start > 0 { - out.normal_text.push_str(&self.buffer[..start]); - } - // Keep the opener bytes in the buffer; the in-block branch needs - // them as the block's leading content for the v1 parser and for - // next-opener detection. - self.buffer.drain(..start); - self.block = Some(kind); +#[derive(Default)] +struct Gemma4InvokeBoundary { + progress: Gemma4InvokeProgress, + candidate: String, + resync_cursor: usize, + resync_in_string: bool, + resync_candidate: Option<(usize, usize)>, +} + +impl InvokeBoundary for Gemma4InvokeBoundary { + fn end_append( + &mut self, + _candidate: &str, + append: &str, + flush: bool, + _tool_index: usize, + ) -> Option { + self.candidate.push_str(append); + self.progress.end(&self.candidate, flush) + } + + fn opens(&self, text: &str, at: usize) -> bool { + opens_bare_call(text, at) + } + + fn holdback(&self, text: &str) -> usize { + partial_bare_opener_suffix_len(text) + } + + fn resync(&mut self, input: &str, flush: bool, _tool_index: usize) -> Option { + if self.resync_cursor > input.len() { + self.resync_cursor = 0; + self.resync_in_string = false; + self.resync_candidate = None; + } + while self.resync_cursor < input.len() { + let cursor = self.resync_cursor; + let rest = &input[cursor..]; + if STRING_DELIM.starts_with(rest) && rest.len() < STRING_DELIM.len() { + return None; + } + if TOOL_CALL_START.starts_with(rest) && rest.len() < TOOL_CALL_START.len() { + return None; + } + if rest.starts_with(STRING_DELIM) { + self.resync_in_string = !self.resync_in_string; + self.resync_cursor += STRING_DELIM.len(); + count_boundary_bytes(STRING_DELIM.len()); + continue; + } + if !self.resync_in_string && cursor > 0 && rest.starts_with(TOOL_CALL_START) { + let after_marker = &rest[TOOL_CALL_START.len()..]; + if CALL_PREFIX.starts_with(after_marker) { + return None; } - None => { - // No complete opener present. Hold back a trailing partial - // opener (a `<|tool_call>` start marker OR a `call:` token that - // may still grow into a recoverable bare call) split across this - // chunk boundary. At flush, an in-progress bare call that never - // completed is DROPPED (truncation parity) rather than leaked as - // normal_text; its preceding prose is still emitted. - let pending_bare = pending_bare_call_suffix_len(&self.buffer); - let keep = if flush { - pending_bare - } else { - self.opener_holdback_len() - }; - let emit_len = self.buffer.len().saturating_sub(keep); - if emit_len > 0 { - out.normal_text.push_str(&self.buffer[..emit_len]); - self.buffer.drain(..emit_len); + if let Some(after_prefix) = after_marker.strip_prefix(CALL_PREFIX) + && after_prefix.find('{').is_none() + && after_prefix + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.')) + { + return None; + } + if let Some(after_prefix) = after_marker.strip_prefix(CALL_PREFIX) + && let Some(name_len) = after_prefix + .char_indices() + .find_map(|(at, ch)| (ch == '{' && at > 0).then_some(at)) + && after_prefix[..name_len] + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.')) + { + self.resync_candidate = Some((cursor, 1)); + let consumed = TOOL_CALL_START.len() + CALL_PREFIX.len() + name_len + 1; + self.resync_cursor += consumed; + count_boundary_bytes(consumed); + continue; + } + } + if let Some((start, depth)) = self.resync_candidate { + if depth == 0 { + if rest.starts_with(TOOL_CALL_END) { + return Some(start); + } + if TOOL_CALL_END.starts_with(rest) { + return None; } - if flush && pending_bare > 0 { - tracing::warn!( - why = "gemma4_incomplete_tool_call", - next_index = self.next_index, - "Gemma 4 stream truncated an incomplete bare call at EOF; dropping it (v1 parity)" - ); - self.buffer.clear(); + if !rest.chars().next()?.is_whitespace() { + self.resync_candidate = None; } - break; + } else if !self.resync_in_string { + self.resync_candidate = match rest.chars().next()? { + '{' => Some((start, depth + 1)), + '}' => Some((start, depth - 1)), + _ => self.resync_candidate, + }; } } + let consumed = rest.chars().next()?.len_utf8(); + self.resync_cursor += consumed; + count_boundary_bytes(consumed); } - - Ok(out) + flush + .then(|| { + self.resync_candidate + .and_then(|(start, depth)| (depth == 0).then_some(start)) + }) + .flatten() } - /// Position of the next opener strictly after the current leading block, used - /// to bound the end-scan to a single block. For a wrapped block, the current - /// block starts at the leading `<|tool_call>`; for a bare block it starts at - /// the leading `call:`. The next opener is the earliest of a later - /// `<|tool_call>` marker or a later recoverable bare `call:`. - fn next_opener_after_current(&self, kind: Block) -> Option { - let skip = match kind { - Block::Wrapped => TOOL_CALL_START.len(), - Block::Bare => CALL_PREFIX.len(), - }; - if skip > self.buffer.len() { - return None; - } - let tail = &self.buffer[skip..]; - let wrapped = tail.find(TOOL_CALL_START).map(|rel| rel + skip); - // A bare `call:` at the very start of a WRAPPED block's tail is the - // block's OWN body (`<|tool_call>call:...`), not a next call. Without - // this skip, every complete wrapped block truncated its own end-scan - // and detoured through drop + bare-recovery (correct output, but two - // misleading warnings per call and a dead Wrapped emit path). - let mut bare = first_bare_call_opener_in(tail); - if kind == Block::Wrapped && bare == Some(0) { - bare = first_bare_call_opener_in(&tail[CALL_PREFIX.len()..]) - .map(|rel| rel + CALL_PREFIX.len()); - } - let bare = bare.map(|rel| rel + skip); - match (wrapped, bare) { - (Some(w), Some(b)) => Some(w.min(b)), - (Some(w), None) => Some(w), - (None, Some(b)) => Some(b), - (None, None) => None, - } + fn reset(&mut self) { + self.progress.reset(); + self.candidate.clear(); + self.resync_cursor = 0; + self.resync_in_string = false; + self.resync_candidate = None; } +} - /// First boundary `call:` in the buffer that begins a recoverable bare call - /// (its complete `}` close has streamed). A boundary `call:` whose - /// body has not yet completed is NOT returned here (it is held back by - /// `opener_holdback_len` until it completes or EOF drops it), so an in-progress - /// body is never leaked as normal_text. - fn first_bare_call_opener(&self) -> Option { - first_bare_call_opener_in(&self.buffer) - } - - /// Bytes to hold back from the tail when no complete opener is present: the - /// longest of a trailing partial `<|tool_call>` prefix, a trailing boundary - /// `call:`-prefixed run that `detect_tool_call_start_gemma4` thinks may still - /// grow into a tool call (so a bare call whose end marker has not yet arrived - /// is buffered, not leaked), or a trailing run the v1 probe cannot see YET — - /// a partial `call:` prefix or a `call:NAME` tail still awaiting its `{`. - /// Without that last component, streaming in small chunks releases `c`, - /// `ca`, ... as normal_text before `call:` can ever accumulate, and a bare - /// call that single-shot parsing recovers is lost (streamv2.5.b/f/g). - fn opener_holdback_len(&self) -> usize { - let partial_start = start_marker_suffix_len(&self.buffer); - let partial_bare = pending_bare_call_suffix_len(&self.buffer); - let partial_opener = partial_bare_opener_suffix_len(&self.buffer); - partial_start.max(partial_bare).max(partial_opener) - } - - /// Parse one complete call block into a delta, delegating name + value typing - /// to the v1 batch parser. For a wrapped block the input includes its leading - /// `<|tool_call>` start marker; for a bare block it starts at `call:` (the v1 - /// parser's missing-start recovery handles the absent opener). Both end with - /// the closing ``. - fn emit_block( - &mut self, - block: &str, - kind: Block, - out: &mut ToolParseResult, - ) -> anyhow::Result<()> { - let (calls, _content) = try_tool_call_parse_gemma4(block, Some(&self.tools))?; - // A complete-but-malformed block (no `call:NAME{...}` body, e.g. the - // `<|tool_call>nonsense` recovery case) yields zero calls; - // drop it without leaking markup, matching the v1 no-leak contract. - let Some(call) = calls.into_iter().next() else { - tracing::warn!( - why = "gemma4_block_without_call", - "Gemma 4 stream dropped a complete block that produced no call" - ); - return Ok(()); - }; - if kind == Block::Bare { - tracing::warn!( - why = "gemma4_bare_call_recovery", - tool_index = self.next_index, - "Gemma 4 stream recovered a bare call (no <|tool_call> opener) instead of leaking it as normal_text" - ); +fn gemma4_invoke_boundary() -> Box { + Box::new(Gemma4InvokeBoundary::default()) +} + +/// Build the Gemma 4 scanner. ONE construction site, shared by the tool-only +/// parser below and by the unified parser, so the grammar cannot drift between +/// the two surfaces. +pub(crate) fn gemma4_scanner(tools: &[Tool]) -> WrappedBlockScanner { + WrappedBlockScanner::new( + WrappedBlockSpec { + family: "gemma4", + block_starts: vec![TOOL_CALL_START.to_string()], + // Same token as `invoke_end`: closing the call closes the block. + block_ends: vec![TOOL_CALL_END.to_string()], + invoke_start: CALL_PREFIX.to_string(), + invoke_end: TOOL_CALL_END.to_string(), + orphan_markers: vec![TOOL_CALL_END.to_string()], + // Only unconditional structural markers belong here because the + // guided path strips this set as markup. The ambiguous `call:` + // prefix has grammar-aware holdback through `Gemma4InvokeBoundary`. + // The string delimiter is unconditionally structural and must not + // be released when split across a chunk boundary. + holdback_markers: vec![ + TOOL_CALL_START.to_string(), + TOOL_CALL_END.to_string(), + STRING_DELIM.to_string(), + ], + // No outer wrapper survives a recovered bare call, so later narration + // is the user's text again. + bare_recovery_latch: BareRecoveryLatch::Clear, + invoke_latch: InvokeLatch::IfEmitted, + drop_invoke_crossing_block_end: false, + invoke_boundary_factory: Some(InvokeBoundaryFactory::custom(gemma4_invoke_boundary)), + preserve_special_tokens: true, + }, + Gemma4InvokeEmitter { + tools: tools.iter().map(ToolDefinition::from).collect(), + }, + ) +} + +/// Whether the `call:` at `at` opens a real call rather than being the English +/// word: it must sit on an identifier boundary AND look like a call start to the +/// candidate-local grammar probe (`call:NAME{`). +/// +/// Without the second test, "I will call: you tomorrow" would be buffered as an +/// invoke that never closes and then dropped at EOF — losing ordinary prose. +fn opens_bare_call(text: &str, at: usize) -> bool { + is_call_prefix_boundary(text, at) && has_bare_call_body_start_gemma4(&text[at..]) +} + +pub(crate) fn is_gemma_call_prefix_boundary(text: &str, at: usize) -> bool { + is_call_prefix_boundary(text, at) +} + +/// Stream parser for Gemma 4 tool calls. +pub struct Gemma4ToolStreamParser { + scanner: WrappedBlockScanner, +} + +impl Gemma4ToolStreamParser { + pub fn new(tools: &[Tool]) -> Self { + Self { + scanner: gemma4_scanner(tools), } - let arguments = reorder_arguments(&call.function.arguments, &source_key_order(block)); - out.calls.push(ToolCallDelta { - tool_index: self.next_index, - name: Some(call.function.name), - arguments, - }); - self.next_index += 1; - Ok(()) } } @@ -312,108 +351,136 @@ impl ToolParser for Gemma4ToolStreamParser { } fn preserve_special_tokens(&self) -> bool { - true + self.scanner.preserve_special_tokens() } fn push(&mut self, chunk: &str) -> anyhow::Result { - self.buffer.push_str(chunk); - self.drain(false) + self.scanner.push(chunk) } fn finish(&mut self) -> anyhow::Result { - self.drain(true) + self.scanner.finish() } } -/// Longest non-empty proper prefix of the `<|tool_call>` start marker that `text` -/// ends with, so a marker split across chunk boundaries is held back instead of -/// leaked as text. The `<|"|>` string delimiter and `` end marker also -/// start with `<`, but only `<|tool_call>` opens a call from the not-in-block -/// state, so holding back its prefixes is sufficient. -fn start_marker_suffix_len(text: &str) -> usize { - TOOL_CALL_START - .char_indices() - .map(|(idx, _)| idx) - .filter(|idx| *idx > 0 && *idx < TOOL_CALL_START.len()) - .rev() - .find(|&len| text.ends_with(&TOOL_CALL_START[..len])) - .unwrap_or(0) -} +#[cfg(test)] +mod boundary_tests { + use super::*; -/// True if `s` begins (at `idx`) on a `call:` word boundary — the preceding char -/// is not part of an identifier, so `"call:"` at the start of a word counts but -/// the `call:` inside `"recall:..."` does not. Mirrors the v1 parser's -/// `is_call_prefix_boundary`. -fn is_call_boundary(s: &str, idx: usize) -> bool { - idx == 0 - || s[..idx] - .chars() - .next_back() - .is_none_or(|c| !(c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'))) -} + #[test] + fn gemma_exposes_its_request_local_boundary() { + let scanner = gemma4_scanner(&[]); + let factory = scanner + .invoke_boundary_factory() + .expect("Gemma needs a grammar-aware boundary factory"); + let boundary = factory.create(); -/// Position of the first boundary `call:` token in `text` that begins a COMPLETE -/// recoverable bare call (its `}` close has streamed). Reuses the v1 -/// helper `find_tool_call_end_position_gemma4` to confirm the candidate completes -/// (and to reject a `call:` that is just prose, e.g. "I will call: you"). An -/// in-progress bare call (no end marker yet) returns `None` here; it is held back -/// by `pending_bare_call_suffix_len` until it completes or EOF drops it. -fn first_bare_call_opener_in(text: &str) -> Option { - let mut cursor = 0usize; - while let Some(rel) = text[cursor..].find(CALL_PREFIX) { - let idx = cursor + rel; - if is_call_boundary(text, idx) && find_tool_call_end_position_gemma4(&text[idx..]).is_some() - { - return Some(idx); - } - cursor = idx + CALL_PREFIX.len(); + assert!(boundary.opens("call:get_weather{}", 0)); + assert_eq!(boundary.holdback("ordinary text"), 0); } - None -} -/// Bytes to hold back from the tail of `text` for a boundary `call:` run that may -/// still grow into a recoverable bare call. Finds the last boundary `call:` whose -/// suffix `detect_tool_call_start_gemma4` still considers a tool-call start (a -/// complete or in-progress `call:NAME{...}` shape) but whose end marker has not -/// yet arrived (`find_tool_call_end_position_gemma4` is `None`), and returns the -/// number of trailing bytes from that `call:` onward so they are buffered, not -/// leaked. Returns 0 when no such pending bare call is present. -fn pending_bare_call_suffix_len(text: &str) -> usize { - let mut cursor = 0usize; - let mut best = 0usize; - while let Some(rel) = text[cursor..].find(CALL_PREFIX) { - let idx = cursor + rel; - if is_call_boundary(text, idx) { - let suffix = &text[idx..]; - // Hold back only an in-progress bare call (looks like a tool-call start - // but has not completed yet); a completed one is handled as an opener. - if find_tool_call_end_position_gemma4(suffix).is_none() - && detect_tool_call_start_gemma4(suffix) - { - best = best.max(suffix.len()); + #[test] + fn resync_advances_only_over_new_bytes_and_recovers_a_balanced_eof_candidate() { + let mut boundary = Gemma4InvokeBoundary::default(); + let mut input = "<|tool_call>call:broken{note:<|\"|>unterminated".to_string(); + + assert_eq!(boundary.resync(&input, false, 0), None); + let examined_after_first_push = boundary.resync_cursor; + assert_eq!(examined_after_first_push, input.len()); + + input.push_str("<|\"|>}<|tool_call>call:get_weather{city:<|\"|>NYC<|\"|>}"); + assert_eq!(boundary.resync(&input, false, 0), None); + assert_eq!(boundary.resync_cursor, input.len()); + assert!(boundary.resync_cursor > examined_after_first_push); + + // The later candidate is balanced but truncated before the wrapper close. + // At EOF it is the safe forward recovery point, without revisiting bytes + // from either preceding push. + assert_eq!( + boundary.resync(&input, true, 0), + Some(input.rfind(TOOL_CALL_START).unwrap()) + ); + } + + #[test] + fn resync_examines_each_byte_once_across_sequential_chunk_sizes() { + let stream = concat!( + "<|tool_call>call:broken{note:<|\"|>unterminated<|\"|>}", + "<|tool_call>call:get_weather{city:<|\"|>NYC<|\"|>}<|tool_call|>" + ); + + for chunk_size in [1, 4, 16] { + let mut boundary = Gemma4InvokeBoundary::default(); + let mut input = String::new(); + let mut examined = 0; + for chunk in stream.as_bytes().chunks(chunk_size) { + input.push_str(std::str::from_utf8(chunk).expect("ASCII test input")); + let before = boundary.resync_cursor; + let _ = boundary.resync(&input, false, 0); + examined += boundary.resync_cursor - before; } + assert_eq!(examined, stream.len(), "chunk_size={chunk_size}"); + let _ = boundary.resync(&input, true, 0); + + boundary.reset(); + assert_eq!( + boundary.resync("<|tool_call>call:get_weather{}<|tool_call|>", true, 0), + None + ); } - cursor = idx + CALL_PREFIX.len(); } - best +} + +/// Per-invoke typing for Gemma 4: hand the complete `call:NAME{…}` +/// block to the v1 batch parser, then restore source key order. +pub(crate) struct Gemma4InvokeEmitter { + tools: Vec, +} + +impl InvokeEmitter for Gemma4InvokeEmitter { + fn parse_invoke( + &mut self, + invoke: &str, + tool_index: usize, + ) -> anyhow::Result> { + // The scanner already delimited this invoke, so type it directly rather + // than re-scanning: re-discovering bounds is what truncates a value at an + // embedded `` (`I7`), and the span scanner also refuses a call + // missing BOTH its opener and closer — the exact shape of case `5.b`. + let call = parse_one_tool_call_gemma4(invoke, Some(&self.tools))?; + // A complete-but-malformed block (no `call:NAME{...}` body, e.g. the + // `<|tool_call>nonsense` recovery case) yields no call; + // drop it without leaking markup, matching the v1 no-leak contract. + let Some(call) = call else { + tracing::warn!( + why = "gemma4_block_without_call", + "Gemma 4 stream dropped a complete block that produced no call" + ); + return Ok(None); + }; + Ok(Some(ToolCallDelta { + tool_index, + name: Some(call.function.name), + arguments: reorder_arguments(&call.function.arguments, &source_key_order(invoke)), + })) + } } /// Trailing run that may still grow into a recoverable bare call but that the /// v1 probes cannot detect yet: a partial `call:` prefix at a word boundary /// (`c`, `ca`, ..., `call:`), or a complete boundary `call:` followed only by /// identifier characters (the function name, awaiting its `{`). Once the `{` -/// arrives, `detect_tool_call_start_gemma4` takes over via -/// `pending_bare_call_suffix_len`. Held-back bytes are flushed on the next -/// chunk (or at EOF), so the concatenated normal_text is unchanged — only its -/// chunk boundaries shift. +/// arrives, the scanner's candidate-local opener probe recognizes it directly. +/// Held-back bytes are flushed on the next chunk (or at EOF), so the +/// concatenated output is unchanged — only its chunk boundaries shift. fn partial_bare_opener_suffix_len(text: &str) -> usize { for len in (1..=CALL_PREFIX.len()).rev() { - if text.ends_with(&CALL_PREFIX[..len]) && is_call_boundary(text, text.len() - len) { + if text.ends_with(&CALL_PREFIX[..len]) && is_call_prefix_boundary(text, text.len() - len) { return len; } } if let Some(idx) = text.rfind(CALL_PREFIX) - && is_call_boundary(text, idx) + && is_call_prefix_boundary(text, idx) && text[idx + CALL_PREFIX.len()..] .chars() .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.')) @@ -680,6 +747,81 @@ mod tests { assert_eq!(merged.calls[0].arguments, r#"{"location":"NYC"}"#); } + #[test] + fn complete_body_missing_end_marker_is_recovered_at_every_split() { + // 5.b at every chunk boundary: the body balances but the close marker + // never streams, so `finish` must recover it regardless of split point. + let input = "<|tool_call>call:get_weather{location:<|\"|>NYC<|\"|>}"; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + let out = parse_chunks(&weather_tools(), &[&input[..split], &input[split..]]); + assert_eq!(out.normal_text, "", "split={split}"); + let merged = out.coalesce_calls(); + assert_eq!(merged.calls.len(), 1, "split={split}"); + assert_eq!( + merged.calls[0].name.as_deref(), + Some("get_weather"), + "split={split}" + ); + assert_eq!( + merged.calls[0].arguments, r#"{"location":"NYC"}"#, + "split={split}" + ); + } + } + + #[test] + fn later_balanced_call_with_trailing_whitespace_is_recovered_at_eof() { + let input = concat!( + "<|tool_call>call:broken{note:<|\"|>unterminated", + "<|\"|>}<|tool_call>call:get_weather{location:<|\"|>NYC<|\"|>} \n\t" + ); + for chunk_size in [1, 4, 16] { + let chunks = input + .as_bytes() + .chunks(chunk_size) + .map(|chunk| std::str::from_utf8(chunk).expect("ASCII test input")) + .collect::>(); + let out = parse_chunks(&weather_tools(), &chunks); + assert_eq!(out.normal_text, " \n\t", "chunk_size={chunk_size}"); + let merged = out.coalesce_calls(); + assert_eq!(merged.calls.len(), 1, "chunk_size={chunk_size}"); + assert_eq!(merged.calls[0].name.as_deref(), Some("get_weather")); + assert_eq!(merged.calls[0].arguments, r#"{"location":"NYC"}"#); + } + } + + #[test] + fn mismatched_close_after_narration_is_rejected_at_every_split() { + let input = "<|tool_call>call:get_weather{location:<|\"|>NYC<|\"|>} trailing "; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + let out = parse_chunks(&weather_tools(), &[&input[..split], &input[split..]]); + assert_eq!(out.normal_text, "", "split={split}"); + assert!(out.calls.is_empty(), "split={split}"); + } + } + + #[test] + fn narrated_call_missing_both_wrappers_is_not_dispatched_at_every_split() { + let input = "for example, call:get_weather{location:<|\"|>NYC<|\"|>}"; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + let out = parse_chunks(&weather_tools(), &[&input[..split], &input[split..]]); + assert_eq!(out.normal_text, "for example, ", "split={split}"); + assert!(out.calls.is_empty(), "split={split}"); + } + } + #[test] fn drops_call_truncated_mid_value() { let out = parse_chunks( @@ -690,6 +832,23 @@ mod tests { assert!(out.calls.is_empty()); } + #[test] + fn truncated_mid_value_call_is_dropped_at_every_split() { + // EOF-mid-value truncation is unrecoverable at every chunk boundary: the + // string value never closes, so the whole call is dropped with no leak, + // regardless of where the stream happens to be cut. + let input = "<|tool_call>call:get_weather{location:<|\"|>N"; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + let out = parse_chunks(&weather_tools(), &[&input[..split], &input[split..]]); + assert_eq!(out.normal_text, "", "split={split}"); + assert!(out.calls.is_empty(), "split={split}"); + } + } + #[test] fn keeps_complete_call_drops_truncated_tail() { // First call complete, second truncated mid-value: keep the first. @@ -812,6 +971,33 @@ mod tests { assert_eq!(merged.calls[0].arguments, r#"{"location":"NYC"}"#); } + #[test] + fn recovers_bare_call_without_opener_at_every_split() { + // streamv2.5.b at every chunk boundary: a bare `call:NAME{...}` + // with no `<|tool_call>` opener must recover as a call, never leak as text, + // no matter where the stream happens to be cut. + let input = "call:get_weather{location:<|\"|>NYC<|\"|>}"; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + let out = parse_chunks(&weather_tools(), &[&input[..split], &input[split..]]); + assert_eq!(out.normal_text, "", "split={split}"); + let merged = out.coalesce_calls(); + assert_eq!(merged.calls.len(), 1, "split={split}"); + assert_eq!( + merged.calls[0].name.as_deref(), + Some("get_weather"), + "split={split}" + ); + assert_eq!( + merged.calls[0].arguments, r#"{"location":"NYC"}"#, + "split={split}" + ); + } + } + #[test] fn recovers_bare_call_keeps_prefix_prose() { // streamv2.5.g: genuine prose precedes a bare `call:` (no opener). The prose diff --git a/parsers/v2/src/tool_calling/kimi_k2.rs b/parsers/v2/src/tool_calling/kimi_k2.rs index 5b7e0cc49..118ecbe15 100644 --- a/parsers/v2/src/tool_calling/kimi_k2.rs +++ b/parsers/v2/src/tool_calling/kimi_k2.rs @@ -35,8 +35,8 @@ //! malformed payloads, which is exactly what the fixtures expect. use crate::tool_calling::scan::{ - BareRecoveryLatch, InvokeEmitter, InvokeLatch, InvokeScan, WrappedBlockScanner, - WrappedBlockSpec, find_first_outside_strings, json_value_end, + BareRecoveryLatch, InvokeBoundaryFactory, InvokeEmitter, InvokeLatch, InvokeScan, + WrappedBlockScanner, WrappedBlockSpec, find_first_outside_strings, json_value_end, }; use crate::tool_calling::v1core::{ KimiK2ParserConfig, ToolDefinition, try_tool_call_parse_kimi_k2, @@ -462,6 +462,7 @@ const KIMI_INVOKE_SCAN: InvokeScan = InvokeScan { end: kimi_invoke_end, opens: kimi_invoke_opens, holdback: kimi_invoke_holdback, + resync: None, }; fn spec(config: &KimiK2ParserConfig) -> WrappedBlockSpec { @@ -493,7 +494,24 @@ fn spec(config: &KimiK2ParserConfig) -> WrappedBlockSpec { drop_invoke_crossing_block_end: true, // Every wrapped family's markers are special tokens today. preserve_special_tokens: true, - invoke_scan: Some(KIMI_INVOKE_SCAN), + invoke_boundary_factory: Some(InvokeBoundaryFactory::stateless(KIMI_INVOKE_SCAN)), + } +} + +#[cfg(test)] +mod boundary_tests { + use super::*; + + #[test] + fn kimi_exposes_its_stateless_boundary_as_family_metadata() { + let scanner = kimi_k2_scanner(&[]); + let factory = scanner + .invoke_boundary_factory() + .expect("Kimi needs grammar-aware boundary callbacks"); + let boundary = factory.create(); + + assert!(boundary.opens(CALL_START, 0)); + assert_eq!(boundary.holdback("ordinary text"), 0); } } diff --git a/parsers/v2/src/tool_calling/scan.rs b/parsers/v2/src/tool_calling/scan.rs index 08d4229eb..aed1f24b9 100644 --- a/parsers/v2/src/tool_calling/scan.rs +++ b/parsers/v2/src/tool_calling/scan.rs @@ -17,12 +17,13 @@ //! * [`reorder_arguments`] — source-order argument reserialization for the //! families that delegate typing to a v1core batch parser (which builds //! arguments from a `HashMap` with nondeterministic key order). -//! * [`WrappedBlockScanner`] — the whole drain loop for the four families +//! * [`WrappedBlockScanner`] — the whole drain loop for the five families //! whose grammar is `BLOCK_START (INVOKE .. INVOKE_END)* BLOCK_END` with a -//! bare-invoke back-off: qwen3_coder, minimax_m2, minimax_m3, kimi_k2. +//! bare-invoke back-off: qwen3_coder, minimax_m2, minimax_m3, kimi_k2, gemma4. +//! Gemma uses [`InvokeScan`] to supply grammar-aware opener, end, and holdback +//! decisions where static markers cannot distinguish structure from data. //! dsml (incremental invoke-header state), glm47 (identifier-anchored bare -//! recovery), and gemma4 (brace/string-aware end scanning) keep bespoke -//! drains and share the primitives. +//! recovery) keep bespoke drains and share the primitives. //! //! Behavior differences between the wrapped families are explicit //! [`WrappedBlockSpec`] fields, not silent copy drift — e.g. MiniMax M2 @@ -263,9 +264,6 @@ pub(crate) enum InvokeLatch { /// Grammar-aware overrides for locating invokes when marker-only scanning is /// insufficient. /// -/// Gemma 4 needs all three answers because its string-delimited values may -/// contain both `call:` and `` as data. Keeping them on one hook -/// prevents opener, end, and holdback ownership from drifting apart. #[derive(Clone, Copy)] pub(crate) struct InvokeScan { /// End of the invoke that begins at byte zero, just past its closer. `flush` @@ -278,6 +276,80 @@ pub(crate) struct InvokeScan { pub opens: fn(text: &str, at: usize) -> bool, /// Additional trailing bytes to retain while an opener is still arriving. pub holdback: fn(text: &str) -> usize, + /// Find a later complete wrapped invoke after the current one proved + /// malformed. Families with ambiguous string syntax own this scan so a + /// block marker inside argument data cannot become a resynchronization + /// point. The hook must inspect `text` in linear time. + pub resync: Option Option>, +} + +/// Immutable family recipe for a request-local invoke boundary. +#[derive(Clone, Copy)] +pub(crate) enum InvokeBoundaryFactory { + Stateless(InvokeScan), + Custom(fn() -> Box), +} + +impl InvokeBoundaryFactory { + pub(crate) const fn stateless(stateless: InvokeScan) -> Self { + Self::Stateless(stateless) + } + + pub(crate) fn create(self) -> Box { + match self { + Self::Stateless(scan) => Box::new(StatelessInvokeBoundary { scan }), + Self::Custom(create) => create(), + } + } + + pub(crate) const fn custom(create: fn() -> Box) -> Self { + Self::Custom(create) + } +} + +/// Family-owned request-local invoke-boundary contract. +/// +pub(crate) trait InvokeBoundary: Send { + /// Advance the bound candidate with newly appended bytes only. + fn end_append( + &mut self, + candidate: &str, + append: &str, + flush: bool, + tool_index: usize, + ) -> Option; + fn opens(&self, text: &str, at: usize) -> bool; + fn holdback(&self, text: &str) -> usize; + fn resync(&mut self, text: &str, flush: bool, tool_index: usize) -> Option; + fn reset(&mut self) {} +} + +struct StatelessInvokeBoundary { + scan: InvokeScan, +} + +impl InvokeBoundary for StatelessInvokeBoundary { + fn end_append( + &mut self, + candidate: &str, + _append: &str, + flush: bool, + tool_index: usize, + ) -> Option { + (self.scan.end)(candidate, flush, tool_index) + } + + fn opens(&self, text: &str, at: usize) -> bool { + (self.scan.opens)(text, at) + } + + fn holdback(&self, text: &str) -> usize { + (self.scan.holdback)(text) + } + + fn resync(&mut self, text: &str, _flush: bool, tool_index: usize) -> Option { + self.scan.resync.and_then(|resync| resync(text, tool_index)) + } } /// Declarative description of one wrapped-invoke grammar. @@ -304,8 +376,9 @@ pub(crate) struct WrappedBlockSpec { /// the `invoke_end` means the call is malformed — drop it and close the /// block (Kimi K2's mismatched-fences rule). pub drop_invoke_crossing_block_end: bool, - /// Grammar-aware invoke location; `None` uses the marker-only rules. - pub invoke_scan: Option, + /// Optional family-owned boundary capability. `None` preserves the + /// marker-only path with no boundary allocation. + pub invoke_boundary_factory: Option, /// Whether a decoder must keep tokenizer special tokens so this grammar's /// markers survive to the parser. /// @@ -543,10 +616,15 @@ pub(crate) struct WrappedBlockScanner { resume_reasoning: bool, suppress_normal_text: bool, next_index: usize, + invoke_boundary: Option>, + invoke_boundary_len: usize, } impl WrappedBlockScanner { pub(crate) fn new(spec: WrappedBlockSpec, emitter: E) -> Self { + let invoke_boundary = spec + .invoke_boundary_factory + .map(InvokeBoundaryFactory::create); Self { spec, emitter, @@ -561,6 +639,8 @@ impl WrappedBlockScanner { resume_reasoning: false, suppress_normal_text: false, next_index: 0, + invoke_boundary, + invoke_boundary_len: 0, } } @@ -615,8 +695,8 @@ impl WrappedBlockScanner { &self.spec.invoke_start } - pub(crate) fn invoke_scan(&self) -> Option { - self.spec.invoke_scan + pub(crate) fn invoke_boundary_factory(&self) -> Option { + self.spec.invoke_boundary_factory } /// The model-emitted id for a call at `tool_index`, delegated to the @@ -714,6 +794,7 @@ impl WrappedBlockScanner { self.suppress_normal_text = false; self.next_index = 0; self.emitter.reset(); + self.reset_invoke_boundary(); pending } @@ -722,15 +803,22 @@ impl WrappedBlockScanner { self.spec.block_ends.contains(&self.spec.invoke_end) } + fn reset_invoke_boundary(&mut self) { + if let Some(boundary) = self.invoke_boundary.as_mut() { + boundary.reset(); + } + self.invoke_boundary_len = 0; + } + /// Find the next real invoke opener, applying the family hook when present. fn find_invoke_start(&self, text: &str) -> Option { - let Some(scan) = self.spec.invoke_scan else { + let Some(boundary) = self.invoke_boundary.as_ref() else { return text.find(self.spec.invoke_start.as_str()); }; let mut cursor = 0; while let Some(relative) = text[cursor..].find(self.spec.invoke_start.as_str()) { let at = cursor + relative; - if (scan.opens)(text, at) { + if boundary.opens(text, at) { return Some(at); } cursor = at + self.spec.invoke_start.len(); @@ -739,15 +827,30 @@ impl WrappedBlockScanner { } /// Offset just past the closer of the invoke beginning at byte zero. - fn invoke_end_at(&self, text: &str, flush: bool) -> Option { - match self.spec.invoke_scan { - Some(scan) => (scan.end)(text, flush, self.next_index), - None => text + fn invoke_end_at(&mut self, flush: bool) -> Option { + match self.invoke_boundary.as_mut() { + Some(boundary) => { + let append = &self.buffer[self.invoke_boundary_len..]; + self.invoke_boundary_len = self.buffer.len(); + boundary.end_append(&self.buffer, append, flush, self.next_index) + } + None => self + .buffer .find(self.spec.invoke_end.as_str()) .map(|position| position + self.spec.invoke_end.len()), } } + /// Find a later complete block that can safely restart scanning after the + /// current invoke proved malformed. The new block must begin with a real, + /// complete invoke; a marker-looking string inside the malformed payload + /// cannot satisfy both grammar checks. + fn resync_block_start(&mut self, flush: bool) -> Option { + self.invoke_boundary + .as_mut()? + .resync(&self.buffer, flush, self.next_index) + } + /// Bytes a reasoning opener occupies at `at`, structural label included. fn opener_len_at(&self, at: usize, flush: bool) -> Option { let reasoning = self.reasoning.as_ref()?; @@ -802,9 +905,9 @@ impl WrappedBlockScanner { }) .unwrap_or_default(); let invoke = self - .spec - .invoke_scan - .map(|scan| (scan.holdback)(&self.buffer)) + .invoke_boundary + .as_ref() + .map(|boundary| boundary.holdback(&self.buffer)) .unwrap_or_default(); regular .max(reasoning) @@ -950,6 +1053,7 @@ impl WrappedBlockScanner { // block re-enters `in_block` and re-suppresses its markup. // Matches the v1 batch parsers (cases 8.b/8.c/8.d). self.buffer.drain(..end_pos + end_len); + self.reset_invoke_boundary(); self.uncommitted_block.clear(); self.in_block = false; self.suppress_normal_text = false; @@ -973,8 +1077,23 @@ impl WrappedBlockScanner { if start > 0 { self.uncommitted_block.push_str(&self.buffer[..start]); self.buffer.drain(..start); + self.reset_invoke_boundary(); } - let Some(end) = self.invoke_end_at(&self.buffer, flush) else { + let Some(end) = self.invoke_end_at(flush) else { + if let Some(next_block) = self.resync_block_start(flush) { + tracing::warn!( + why = %format!("{}_resynchronized_after_incomplete_invoke", self.spec.family), + skipped_bytes = next_block, + "stream skipped a malformed invoke and resumed at the next complete block" + ); + self.buffer.drain(..next_block); + self.reset_invoke_boundary(); + self.uncommitted_block.clear(); + self.in_block = false; + self.suppress_normal_text = false; + self.in_reasoning = std::mem::take(&mut self.resume_reasoning); + continue; + } if flush { // Reviewer-caught regression: `invoke_end_at` // returning `None` at flush does NOT always mean @@ -1006,6 +1125,7 @@ impl WrappedBlockScanner { "stream dropped invoke with no evidence it ever closed before the block end" ); self.buffer.drain(..be_pos + be_len); + self.reset_invoke_boundary(); self.uncommitted_block.clear(); self.in_block = false; self.suppress_normal_text = false; @@ -1065,6 +1185,7 @@ impl WrappedBlockScanner { // documented recovery contract was false for the only shipped family. let emitted = self.emitter.parse_invoke(&invoke, self.next_index)?; self.buffer.drain(..end); + self.reset_invoke_boundary(); if let Some(delta) = emitted { out.push_call(delta); self.next_index += 1; @@ -1148,6 +1269,7 @@ impl WrappedBlockScanner { Marker::Block(blen) => { self.uncommitted_block.push_str(&self.buffer[..blen]); self.buffer.drain(..blen); + self.reset_invoke_boundary(); self.in_block = true; self.suppress_normal_text = true; } @@ -1163,7 +1285,11 @@ impl WrappedBlockScanner { Marker::BareInvoke => { // A bare invoke (no wrapper) is recovered only once its close // has streamed; otherwise wait for more input. - let Some(end) = self.invoke_end_at(&self.buffer, flush) else { + // A bare invoke already lacks its opener, so it may recover + // only after a real closer arrives. Passing `flush` here used + // to combine missing-start and missing-end recovery and turn + // narrated syntax into a dispatched call. + let Some(end) = self.invoke_end_at(false) else { if flush { tracing::warn!( why = %format!("{}_incomplete_bare_invoke", self.spec.family), @@ -1177,6 +1303,7 @@ impl WrappedBlockScanner { // Emit before consuming — same recovery contract as the wrapped site. let emitted = self.emitter.parse_invoke(&invoke, self.next_index)?; self.buffer.drain(..end); + self.reset_invoke_boundary(); if let Some(delta) = emitted { tracing::warn!( why = %format!("{}_bare_invoke_recovery", self.spec.family), @@ -1242,7 +1369,7 @@ pub(crate) mod test_support { bare_recovery_latch: BareRecoveryLatch::Set, invoke_latch: InvokeLatch::IfEmitted, drop_invoke_crossing_block_end: false, - invoke_scan: None, + invoke_boundary_factory: None, preserve_special_tokens: true, }, FailOnBoom, @@ -1292,6 +1419,41 @@ mod tests { use super::test_support::failing_scanner; use super::*; + #[test] + fn invoke_boundary_factory_constructs_and_dispatches_stateless_callbacks() { + fn end(text: &str, flush: bool, tool_index: usize) -> Option { + (flush && tool_index == 2).then_some(text.len()) + } + fn opens(text: &str, at: usize) -> bool { + text[at..].starts_with("open") + } + fn holdback(text: &str) -> usize { + text.len().min(3) + } + fn resync(text: &str, tool_index: usize) -> Option { + (tool_index == 2).then(|| text.find("resume")).flatten() + } + + let factory = InvokeBoundaryFactory::stateless(InvokeScan { + end, + opens, + holdback, + resync: Some(resync), + }); + let mut boundary = factory.create(); + + assert_eq!(boundary.end_append("call", "call", false, 2), None); + assert_eq!(boundary.end_append("call", "", true, 2), Some(4)); + assert!(boundary.opens("open-call", 0)); + assert_eq!(boundary.holdback("abcdef"), 3); + assert_eq!(boundary.resync("skip resume", false, 2), Some(5)); + } + + #[test] + fn marker_only_scanner_has_no_invoke_boundary_factory() { + assert!(failing_scanner().invoke_boundary_factory().is_none()); + } + #[test] fn json_value_end_rejects_mismatched_bracket_types() { // A numeric depth counter treats `}`/`]` as interchangeable and diff --git a/parsers/v2/src/tool_calling/v1core/gemma4/mod.rs b/parsers/v2/src/tool_calling/v1core/gemma4/mod.rs index 1e01a9fe2..e32ee2ef4 100644 --- a/parsers/v2/src/tool_calling/v1core/gemma4/mod.rs +++ b/parsers/v2/src/tool_calling/v1core/gemma4/mod.rs @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Vendored Gemma-4 batch extractor. +//! Vendored Gemma-4 grammar helpers. mod parser; pub use parser::{ - detect_tool_call_start_gemma4, find_tool_call_end_position_gemma4, try_tool_call_parse_gemma4, + has_bare_call_body_start_gemma4, is_call_prefix_boundary, parse_one_tool_call_gemma4, }; diff --git a/parsers/v2/src/tool_calling/v1core/gemma4/parser.rs b/parsers/v2/src/tool_calling/v1core/gemma4/parser.rs index 5f07c2b90..f37caddfe 100644 --- a/parsers/v2/src/tool_calling/v1core/gemma4/parser.rs +++ b/parsers/v2/src/tool_calling/v1core/gemma4/parser.rs @@ -21,7 +21,6 @@ pub(crate) const TOOL_CALL_START: &str = "<|tool_call>"; pub(crate) const TOOL_CALL_END: &str = ""; pub(crate) const STRING_DELIM: &str = "<|\"|>"; pub(crate) const CALL_PREFIX: &str = "call:"; - fn parse_gemma_call_parts( name: &str, args_raw: &str, @@ -131,7 +130,7 @@ fn parse_recoverable_call_at( None } -fn is_call_prefix_boundary(input: &str, idx: usize) -> bool { +pub fn is_call_prefix_boundary(input: &str, idx: usize) -> bool { idx == 0 || input[..idx] .chars() @@ -139,352 +138,46 @@ fn is_call_prefix_boundary(input: &str, idx: usize) -> bool { .is_none_or(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))) } -fn find_call_prefix_at_boundary(input: &str, from: usize) -> Option { - let mut cursor = from; - while cursor < input.len() { - let rel = input[cursor..].find(CALL_PREFIX)?; - let idx = cursor + rel; - if is_call_prefix_boundary(input, idx) { - return Some(idx); - } - cursor = idx + CALL_PREFIX.len(); - } - None -} - -/// Detect whether `chunk` contains the start of a Gemma 4 tool call, including -/// partial-prefix matches at the chunk boundary so streaming pipelines can hold -/// off emitting bytes that may belong to a tool-call marker. -pub fn detect_tool_call_start_gemma4(chunk: &str) -> bool { - if chunk.contains(TOOL_CALL_START) { - return true; - } - - let mut cursor = 0usize; - while let Some(idx) = find_call_prefix_at_boundary(chunk, cursor) { - let candidate = &chunk[idx..]; - if parse_recoverable_call_at(candidate, true, true).is_some() - || has_bare_call_body_start(candidate) - { - return true; - } - cursor = idx + CALL_PREFIX.len(); - } - - for i in 1..TOOL_CALL_START.len() { - if TOOL_CALL_START.is_char_boundary(i) && chunk.ends_with(&TOOL_CALL_START[..i]) { - return true; - } - } - - false +/// Whether the candidate at byte zero has Gemma's `call:NAME{` prefix. +/// +/// This probe is deliberately candidate-local: the first non-identifier byte +/// decides the result. Searching for any later `{` lets ordinary `call:` prose +/// borrow the body of a later real call and makes a suffix-wide scan quadratic. +pub fn has_bare_call_body_start_gemma4(input: &str) -> bool { + bare_call_body_start_probe(input).0 } -fn has_bare_call_body_start(input: &str) -> bool { +fn bare_call_body_start_probe(input: &str) -> (bool, usize) { let Some(after_prefix) = input.strip_prefix(CALL_PREFIX) else { - return false; - }; - let Some(open_brace) = after_prefix.find('{') else { - return false; + return (false, 0); }; - if open_brace == 0 { - return false; - } - after_prefix[..open_brace] - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.')) -} - -/// Returns the position immediately after the last *complete* tool-call match -/// in `chunk`, or `None` if no complete call has arrived yet (caller should -/// keep accumulating). The string-aware balanced scanner requires `}` -/// adjacency at the real object close, so a bare `` literal embedded -/// inside a `<|"|>` string value does not false-trigger a "section complete" -/// signal here — matches upstream. -pub fn find_tool_call_end_position_gemma4(chunk: &str) -> Option { - let mut cursor = 0usize; - let mut last_end = None; - - while cursor < chunk.len() { - let next_start = chunk[cursor..] - .find(TOOL_CALL_START) - .map(|rel| (cursor + rel, false, TOOL_CALL_START.len())); - let next_bare = - find_call_prefix_at_boundary(chunk, cursor).map(|idx| (idx, true, CALL_PREFIX.len())); - let Some((rel_start, allow_missing_start, marker_len)) = [next_start, next_bare] - .into_iter() - .flatten() - .min_by_key(|(idx, _, _)| *idx) - else { - break; - }; - - if let Some((_, _, consumed)) = - parse_recoverable_call_at(&chunk[rel_start..], allow_missing_start, false) - { - let mut end = rel_start + consumed; - while chunk[end..].starts_with(TOOL_CALL_END) { - end += TOOL_CALL_END.len(); - } - last_end = Some(end); - cursor = end; - } else { - cursor = rel_start + marker_len; + let mut saw_name = false; + let mut inspected = 0; + for ch in after_prefix.chars() { + inspected += 1; + if ch == '{' { + return (saw_name, inspected); } - } - - last_end -} - -fn push_recovered_call( - calls: &mut Vec, - first_tool_start: &mut Option, - removed_spans: &mut Vec<(usize, usize)>, - absolute_start: usize, - recovered: (&str, &str, usize), - tools: Option<&[ToolDefinition]>, - reason: &'static str, -) -> anyhow::Result<()> { - if first_tool_start.is_none_or(|idx| absolute_start < idx) { - *first_tool_start = Some(absolute_start); - } - // Record the byte span of the recovered call so the normal_text assembler - // can strip exactly this markup and keep the surrounding natural text. - removed_spans.push((absolute_start, absolute_start + recovered.2)); - tracing::warn!( - why = reason, - recovered_calls = 1, - recovered_bytes = recovered.2, - "gemma4 recovery: recovered complete call body from damaged wrapper" - ); - calls.push(parse_gemma_call_parts(recovered.0, recovered.1, tools)?); - Ok(()) -} - -fn recover_calls_in_span( - span: &str, - span_offset: usize, - allow_missing_end: bool, - tools: Option<&[ToolDefinition]>, - calls: &mut Vec, - first_tool_start: &mut Option, - removed_spans: &mut Vec<(usize, usize)>, -) -> anyhow::Result<()> { - let mut cursor = 0usize; - - while cursor < span.len() { - let next_start = span[cursor..] - .find(TOOL_CALL_START) - .map(|rel| (cursor + rel, false, TOOL_CALL_START.len())); - let next_bare = - find_call_prefix_at_boundary(span, cursor).map(|idx| (idx, true, CALL_PREFIX.len())); - let Some((rel_start, allow_missing_start, marker_len)) = [next_start, next_bare] - .into_iter() - .flatten() - .min_by_key(|(idx, _, _)| *idx) - else { - break; - }; - - let parsed = parse_recoverable_call_at( - &span[rel_start..], - allow_missing_start, - allow_missing_end && !allow_missing_start, - ); - - if let Some(recovered) = parsed { - let reason = if allow_missing_start { - "missing_start_recovery" - } else { - "missing_end_recovery" - }; - push_recovered_call( - calls, - first_tool_start, - removed_spans, - span_offset + rel_start, - recovered, - tools, - reason, - )?; - cursor = rel_start + recovered.2; - } else { - cursor = rel_start + marker_len; + if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.') { + saw_name = true; + continue; } + return (false, inspected); } - - Ok(()) + (false, inspected) } -/// Parse a Gemma 4 model response into structured tool calls + leftover text. -/// -/// Returns `(parsed_tool_calls, normal_text_content)`. `normal_text` is the -/// model text with each complete tool-call block (start marker `<|tool_call>` -/// through end marker ``) removed, keeping ALL other text: -/// the prefix before the first call, the text BETWEEN calls, and the text -/// AFTER the last call. Interior whitespace is preserved as-is (only the -/// outermost ends are trimmed, matching the rest of the parser family); only -/// tool-call markup is stripped, natural text is never dropped, and markup -/// never leaks. -/// -/// This is a deliberate divergence from upstream vLLM -/// (`vllm/tool_parsers/gemma4_tool_parser.py::extract_tool_calls`, which keeps -/// only `model_output[:content_end].strip()`, i.e. the prefix before the first -/// marker) and from SGLang (which also drops trailing narration). Dynamo -/// preserves the surrounding narration so a model that sandwiches a tool call -/// between sentences doesn't silently lose the user-visible text. Cases: -/// TOOLCALLING.batch.{2.c, 8.b, 8.c, 8.d}. -/// -/// Malformed / unrecoverable / truncated tool-call markup keeps its no-leak -/// drop behavior: a gap that still contains gemma4 markup tokens after the -/// complete calls are removed is suppressed rather than echoed verbatim. -/// -/// Primary extraction uses the string-aware balanced scanner -/// (`find_balanced_args_end`), so a `}` sequence embedded inside a -/// `<|"|>`-delimited string arg (e.g. a `description` field documenting the -/// tool-call format) is treated as data and does not truncate the call early. -pub fn try_tool_call_parse_gemma4( - message: &str, +/// Type one call whose boundaries have already been resolved by the caller. +pub fn parse_one_tool_call_gemma4( + invoke: &str, tools: Option<&[ToolDefinition]>, -) -> anyhow::Result<(Vec, Option)> { - let mut calls = Vec::new(); - let mut first_tool_start = None; - // Byte spans of every complete (parsed or recovered) tool-call block, used - // to subtract markup from the assembled normal_text below. - let mut removed_spans: Vec<(usize, usize)> = Vec::new(); - // `gap_start` is the end of the last complete call (start of the current - // natural-text gap); `scan` is where we look for the next start marker. - let mut gap_start = 0usize; - let mut scan = 0usize; - - // Primary extraction: walk each `<|tool_call>` start marker and resolve the - // matching end with the string-aware balanced scanner (`find_balanced_args_end` - // via `parse_recoverable_call_at`) rather than a lazy `.*?}` regex. - // A `}` sequence embedded inside a `<|"|>`-delimited string value - // is data, not a terminator, so it must NOT truncate the call — the scanner - // ignores braces/markers while `in_string`, matching upstream vLLM. - while scan < message.len() { - let Some(rel) = message[scan..].find(TOOL_CALL_START) else { - break; - }; - let start = scan + rel; - - // A complete call requires both the start marker and the `` - // end after the balanced args body. - match parse_recoverable_call_at(&message[start..], false, false) { - Some((name, args_raw, consumed)) => { - // Recover any missing-start (bare `call:`) blocks in the gap that - // precedes this complete call, preserving natural text between them. - recover_calls_in_span( - &message[gap_start..start], - gap_start, - false, - tools, - &mut calls, - &mut first_tool_start, - &mut removed_spans, - )?; - first_tool_start.get_or_insert(start); - removed_spans.push((start, start + consumed)); - calls.push(parse_gemma_call_parts(name, args_raw, tools)?); - gap_start = start + consumed; - scan = start + consumed; - } - // This start marker did not resolve to a complete call (e.g. no end - // marker yet, or an invalid name); leave it in the current gap and - // keep scanning past the marker. It is handled by span recovery / the - // no-leak markup drop below. - None => scan = start + TOOL_CALL_START.len(), - } - } - - recover_calls_in_span( - &message[gap_start..], - gap_start, - true, - tools, - &mut calls, - &mut first_tool_start, - &mut removed_spans, - )?; - - // `normal_text` assembly: - // - Success path (≥1 call extracted): concatenate the text outside every - // complete tool-call block — prefix, inter-call narration, and trailing - // narration — preserving whitespace verbatim. A gap that still contains - // gemma4 markup tokens (`<|tool_call>`, ``, `<|"|>`) is a - // malformed / truncated / unrecoverable remnant; it is dropped so - // tool-call markup never leaks into normal_text. - // - Recovery path (zero calls extracted): if the message contains ANY - // gemma4 markup token, return empty — Dynamo intentionally diverges - // from vLLM's exception-fallback (which echoes raw bytes) so tool-call - // markup never leaks on malformed / truncated / orphan-close / no-body - // inputs. Cases flagged by the parity table's `↯`: - // TOOLCALLING.batch.{4.a, 4.b, 4.c, 4.d, 5.a, 5.b, 5.c, 6.c}. - // - Plain-text path (zero calls AND no markup): return the message as-is. - let has_markup = message.contains(TOOL_CALL_START) - || message.contains(TOOL_CALL_END) - || message.contains(STRING_DELIM); - let normal_text = if calls.is_empty() { - if has_markup { - // Recovery: malformed/truncated/orphan-close/no-body shapes. - // Suppress the whole message so tool-call markup doesn't leak. - let preview: String = message.chars().take(120).collect(); - tracing::warn!( - why = "no_calls_with_markup", - stripped_bytes = message.len(), - has_start = message.contains(TOOL_CALL_START), - has_end = message.contains(TOOL_CALL_END), - has_string_delim = message.contains(STRING_DELIM), - "gemma4 strip (recovery): zero calls extracted but gemma4 markup present (<|tool_call>, , <|\"|>); suppressing entire message to prevent leak into normal_text. preview={:?}", - preview - ); - String::new() - } else { - // No markup at all → plain text passes through unchanged. No strip. - message.trim().to_string() - } - } else { - // Success: keep every gap between/around the complete tool-call blocks. - removed_spans.sort_unstable(); - let mut kept = String::new(); - let mut prev_end = 0usize; - for &(start, end) in &removed_spans { - // Spans are non-overlapping and sorted; guard against any pathological - // overlap so we never slice on a stale offset. - if start >= prev_end { - push_natural_text_gap(&mut kept, &message[prev_end..start]); - } - prev_end = prev_end.max(end); - } - push_natural_text_gap(&mut kept, &message[prev_end..]); - // Trim only the outermost ends (interior whitespace stays verbatim) to - // match the dynamo normal_text convention used across this family. - kept.trim().to_string() +) -> anyhow::Result> { + let allow_missing_start = !invoke.starts_with(TOOL_CALL_START); + let Some((name, args_raw, _)) = parse_recoverable_call_at(invoke, allow_missing_start, true) + else { + return Ok(None); }; - - Ok((calls, Some(normal_text))) -} - -/// Append a single inter-call / surrounding gap to the accumulated normal_text. -/// A gap that still contains gemma4 tool-call markup is a malformed / truncated -/// remnant (e.g. an incomplete trailing call that never closed) — drop it so -/// markup never leaks. Natural text is appended verbatim, whitespace included. -fn push_natural_text_gap(kept: &mut String, gap: &str) { - if gap.is_empty() { - return; - } - if gap.contains(TOOL_CALL_START) || gap.contains(TOOL_CALL_END) || gap.contains(STRING_DELIM) { - let preview: String = gap.chars().take(120).collect(); - tracing::warn!( - why = "dropped_markup_gap", - stripped_bytes = gap.len(), - "gemma4 strip (success): dropped a gap that still contained gemma4 markup (malformed/truncated remnant) to prevent leak into normal_text. preview={:?}", - preview - ); - return; - } - kept.push_str(gap); + parse_gemma_call_parts(name, args_raw, tools).map(Some) } // --------------------------------------------------------------------------- @@ -741,31 +434,49 @@ fn parse_number(cur: &mut Cursor) -> anyhow::Result { mod tests { use super::*; - /// A string-typed argument whose *value* literally contains the - /// `}` end-marker sequence must NOT truncate the call there: - /// the terminator inside a `<|"|>`-delimited string is data, and the real - /// close is the `}` that follows the closing string delimiter. - /// - /// This is the CodeRabbit regression: a lazy `.*?}` regex stops - /// at the first `}`, even inside a string, yielding truncated - /// arguments and a too-short removal span. The balanced, string-aware - /// scanner ignores the terminator while `in_string`. #[test] - fn end_marker_sequence_inside_string_arg_does_not_truncate() { - let message = "before <|tool_call>call:doc{note:<|\"|>use } to end<|\"|>} after"; + fn a_body_missing_its_close_marker_is_still_typeable() { + let body = "call:get_weather{city:<|\"|>Paris<|\"|>}"; + assert_eq!( + parse_one_tool_call_gemma4(body, None) + .unwrap() + .expect("recovered") + .function + .arguments, + r#"{"city":"Paris"}"# + ); + } - let (calls, normal_text) = try_tool_call_parse_gemma4(message, None).unwrap(); + #[test] + fn a_mismatched_xml_close_is_not_typeable_as_a_call() { + let input = "call:get_weather{city:<|\"|>Paris<|\"|>}"; + assert!(parse_one_tool_call_gemma4(input, None).unwrap().is_none()); + } - assert_eq!(calls.len(), 1, "exactly one complete call"); - assert_eq!(calls[0].function.name, "doc"); - // Arguments must be COMPLETE — the embedded `}` stays inside - // the string value rather than terminating the call early. - assert_eq!( - calls[0].function.arguments, - r#"{"note":"use } to end"}"# + #[test] + fn narration_does_not_hide_a_later_mismatched_close() { + let input = "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>} trailing "; + assert!(parse_one_tool_call_gemma4(input, None).unwrap().is_none()); + } + + #[test] + fn bare_complete_call_remains_typeable() { + let input = "call:get_weather{city:<|\"|>Paris<|\"|>}"; + assert!(parse_one_tool_call_gemma4(input, None).unwrap().is_some()); + } + + #[test] + fn prose_containing_the_word_call_is_not_a_leading_call() { + assert!( + parse_one_tool_call_gemma4("call: you tomorrow", None) + .unwrap() + .is_none() ); - // Removal span must cover the ENTIRE call block (start marker through the - // real end marker), so surrounding prose survives and no markup leaks. - assert_eq!(normal_text.as_deref(), Some("before after")); + } + + #[test] + fn bare_call_probe_stops_at_the_first_non_identifier() { + let input = format!("call: {}{{", "ordinary prose ".repeat(4096)); + assert_eq!(bare_call_body_start_probe(&input), (false, 1)); } } diff --git a/parsers/v2/src/unified/gemma4.rs b/parsers/v2/src/unified/gemma4.rs new file mode 100644 index 000000000..14e4c5559 --- /dev/null +++ b/parsers/v2/src/unified/gemma4.rs @@ -0,0 +1,892 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Unified parser for the Gemma 4 grammar: the Gemma 4 tool grammar plus the +//! `<|channel>` reasoning channel, in ONE state machine. +//! +//! ```text +//! reasoning: <|channel>thought\n … +//! tool call: <|tool_call>call:NAME{key:<|"|>value<|"|>} +//! everything else: visible content +//! ``` +//! +//! This file is only the grammar wiring. The scan core is the same +//! [`crate::tool_calling::scan::WrappedBlockScanner`] the tool-only +//! `Gemma4ToolStreamParser` runs on — block open/close, bare-`call:` recovery, +//! orphan-close stripping, chunk-boundary holdback and EOF recovery stay a single +//! implementation, so the unified path cannot quietly regress the tool handling +//! the tool-only suite already pins. Value typing likewise delegates to the +//! shared v1 batch parser, so a streamed argument object matches the batch one +//! exactly. The `UnifiedParser` impl itself is generic +//! ([`crate::unified::ScannerUnified`]). +//! +//! Gemma 4 was the hardest family to move because its END MARKER IS DATA: a +//! literal `` inside a `<|"|>`-delimited string value is an argument +//! value, so the plain `find` every other wrapped family uses would cut the value +//! there (`I7`, case `7.b`). The scanner takes a grammar-aware +//! [`crate::tool_calling::scan::InvokeScan`] for exactly that; see +//! `tool_calling/gemma4.rs`. +//! +//! The reasoning side needed the other extension. Gemma 4's opener is a marker +//! PLUS a `thought\n` role label, and the label is OPTIONAL — folding it into +//! `start` parses the corpus but makes a bare `<|channel>` open nothing and leak +//! its markup as visible text. [`ReasoningSpec::start_label`] keeps the label +//! structural and optional (policy `P4`). +//! +//! What the unified path adds is ORDER: a thought between two calls is a second +//! thought in its own position instead of being hoisted into the first. Nesting +//! is asymmetric, exactly as for qwen3 — a call inside a thought is extracted and +//! the thought splits around it (`I3`, case `12.b`), while a channel marker +//! inside a quoted argument value is data and survives byte-exact (`I7`, case +//! `12.a`). + +use crate::tool_calling::gemma4::{gemma4_scanner, is_gemma_call_prefix_boundary}; +use crate::tool_calling::scan::ReasoningSpec; +use crate::tool_calling::traits::Tool; +use crate::unified::{ + GuidedPrefix, GuidedPrefixContext, GuidedRouted, ScannerUnified, UnifiedParser, +}; + +const REASONING_START: &str = "<|channel>"; +const REASONING_END: &str = ""; +/// The role label the tokenizer writes inside the channel, analogous to the +/// `user\n` in `<|turn>user\n`. Structural, and stripped from the thought. +const REASONING_START_LABEL: &str = "thought\n"; + +/// Gemma's lexical `call:` prefix is guided syntax only before an otherwise empty +/// payload and only when it immediately introduces JSON. The generic guided parser +/// supplies the surrounding context; this family owns the literal spelling. +fn guided_call_prefix(context: GuidedPrefixContext<'_>) -> GuidedPrefix { + if !is_gemma_call_prefix_boundary(context.text, context.at) { + return GuidedPrefix::NoMatch; + } + + let suffix = &context.text[context.at..]; + if suffix.len() < "call:".len() && "call:".starts_with(suffix) { + return GuidedPrefix::Pending; + } + let Some(after_prefix) = suffix.strip_prefix("call:") else { + return GuidedPrefix::NoMatch; + }; + if context.followed_by_competing_marker { + return GuidedPrefix::Strip; + } + if !context.outside_reasoning + && after_prefix + .chars() + .next() + .is_some_and(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.')) + { + return GuidedPrefix::Strip; + } + if !context.outside_reasoning + || !context.payload_is_empty + || !context.text[..context.at].trim().is_empty() + { + return GuidedPrefix::NoMatch; + } + match after_prefix.as_bytes().first() { + None => GuidedPrefix::Pending, + Some(b'{') | Some(b'[') => GuidedPrefix::Match, + Some(_) => GuidedPrefix::NoMatch, + } +} + +/// Build the Gemma 4 unified parser for one stream. +pub(crate) fn gemma4_unified(tools: &[Tool]) -> Box { + // `preserve_special_tokens` must match the TOOL-ONLY parser for this same + // grammar (`Gemma4ToolStreamParser` answers `true`): gemma4's markers ARE + // tokenizer special tokens, so a host told `false` would strip them and the + // calls would vanish on the unified path only. + Box::new(GuidedRouted::new( + ScannerUnified::new(gemma4_scanner(tools).with_reasoning(ReasoningSpec { + start: REASONING_START, + end: REASONING_END, + start_label: Some(REASONING_START_LABEL), + // Gemma 4 emits its own `<|channel>`; the template does not pre-fill + // one, so the stream starts in visible content (policy P5). + forced_start: false, + preserve_special_tokens: true, + })) + .with_guided_prefix_policy(guided_call_prefix), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tool_calling::gemma4::{ + Gemma4ToolStreamParser, boundary_examined_bytes, reset_boundary_examined_bytes, + }; + use crate::tool_calling::traits::ToolParser; + use crate::unified::{ + InvalidGuidedPayloadPolicy, UnifiedEvent, UnifiedParserExt, UnifiedParserInit, + UnifiedParserStartingState, UnifiedToolOutputMode, assemble, + }; + + fn weather_tools() -> Vec { + vec![Tool { + name: "get_weather".to_string(), + description: None, + parameters: serde_json::json!({ + "type": "object", + "properties": { "city": { "type": "string" } } + }), + strict: None, + }] + } + + fn tool(name: &str, key: &str) -> Tool { + Tool { + name: name.to_string(), + description: None, + parameters: serde_json::json!({ + "type": "object", + "properties": { key: { "type": "string" } } + }), + strict: None, + } + } + + fn events(tools: &[Tool], chunks: &[&str]) -> Vec { + let mut parser = gemma4_unified(tools); + let mut deltas = Vec::new(); + for chunk in chunks { + deltas.extend(parser.push(chunk).expect("push")); + } + deltas.extend(parser.finish().expect("finish").events); + assemble(&deltas) + } + + fn configured_events( + tools: &[Tool], + starting_state: UnifiedParserStartingState, + tool_output_mode: UnifiedToolOutputMode, + chunks: &[&str], + ) -> Vec { + let mut parser = gemma4_unified(tools); + parser + .initialize_request(UnifiedParserInit { + starting_state, + tool_output_mode, + invalid_guided_payload: InvalidGuidedPayloadPolicy::RecoverAsText, + ..UnifiedParserInit::default() + }) + .expect("initialize"); + let mut deltas = Vec::new(); + for chunk in chunks { + deltas.extend(parser.push(chunk).expect("push")); + } + deltas.extend(parser.finish().expect("finish").events); + assemble(&deltas) + } + + fn reasoning(text: &str) -> UnifiedEvent { + UnifiedEvent::Reasoning { text: text.into() } + } + fn text(text: &str) -> UnifiedEvent { + UnifiedEvent::Text { text: text.into() } + } + fn call(name: &str, arguments: serde_json::Value) -> UnifiedEvent { + UnifiedEvent::ToolCall { + name: name.into(), + arguments, + } + } + + fn one_byte_events( + tools: &[Tool], + starting_state: UnifiedParserStartingState, + tool_output_mode: UnifiedToolOutputMode, + input: &str, + ) -> Vec { + let chunks = input + .as_bytes() + .iter() + .map(|byte| std::str::from_utf8(std::slice::from_ref(byte)).expect("ASCII input")) + .collect::>(); + configured_events(tools, starting_state, tool_output_mode, &chunks) + } + + #[test] + fn native_and_guided_boundary_scans_are_linear_across_one_byte_streams() { + let native = "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>}"; + let guided = "<|channel>thought\nprivate[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}]"; + + reset_boundary_examined_bytes(); + let mut tool_only = Gemma4ToolStreamParser::new(&weather_tools()); + for byte in native.bytes() { + tool_only + .push(&char::from(byte).to_string()) + .expect("tool-only push"); + } + tool_only.finish().expect("tool-only finish"); + assert!( + boundary_examined_bytes() <= native.len() * 2, + "tool-only boundary examined {} bytes for a {}-byte one-byte stream", + boundary_examined_bytes(), + native.len() + ); + + for (mode, input) in [ + (UnifiedToolOutputMode::Native, native), + ( + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + guided, + ), + ] { + reset_boundary_examined_bytes(); + let _ = one_byte_events( + &weather_tools(), + UnifiedParserStartingState::None, + mode, + input, + ); + assert!( + boundary_examined_bytes() <= input.len() * 2, + "boundary examined {} bytes for a {}-byte one-byte stream", + boundary_examined_bytes(), + input.len() + ); + } + } + + #[test] + fn guided_native_looking_candidates_scan_each_byte_once() { + for payload_len in [8 * 1024, 16 * 1024, 32 * 1024] { + let input = format!( + "<|tool_call>call:get_weather{{city:<|\"|>{}}}", + "x".repeat(payload_len) + ); + + reset_boundary_examined_bytes(); + let events = one_byte_events( + &weather_tools(), + UnifiedParserStartingState::None, + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + &input, + ); + + assert!( + events.is_empty(), + "incomplete native candidate must be retained" + ); + assert!( + boundary_examined_bytes() <= input.len() * 2, + "guided boundary examined {} bytes for a {}-byte native-looking candidate", + boundary_examined_bytes(), + input.len() + ); + } + } + + #[test] + fn one_byte_guided_stream_strips_native_envelope_before_reasoning() { + let input = "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>}<|channel>thought\nprivate[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Rome\"}}]"; + assert_eq!( + one_byte_events( + &weather_tools(), + UnifiedParserStartingState::None, + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + input, + ), + vec![ + text("call:get_weather{city:<|\"|>Paris<|\"|>}"), + reasoning("private"), + call("get_weather", serde_json::json!({"city": "Rome"})), + ] + ); + } + + #[test] + fn reasoning_after_a_call_keeps_its_position() { + // The defect the unified parser exists to fix: under the split, both + // thoughts merge into one span ahead of the call. + let out = events( + &weather_tools(), + &[ + "<|channel>thought\nLook it up.", + "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>}", + "<|channel>thought\nNow answer.It's 18C.", + ], + ); + assert_eq!( + out, + vec![ + reasoning("Look it up."), + call("get_weather", serde_json::json!({"city": "Paris"})), + reasoning("Now answer."), + text("It's 18C."), + ] + ); + } + + #[test] + fn the_thought_role_label_is_structural_but_optional() { + // P4: `thought\n` is tokenizer grammar, so it is stripped — but a channel + // that opens WITHOUT it keeps every byte of the thought, including a first + // word that merely starts like the label. Folding the label into `start` + // would leave `<|channel>` opening nothing and leaking as text. + assert_eq!( + events(&weather_tools(), &["<|channel>thought\nstripped"]), + vec![reasoning("stripped")] + ); + assert_eq!( + events(&weather_tools(), &["<|channel>no label here"]), + vec![reasoning("no label here")] + ); + assert_eq!( + events(&weather_tools(), &["<|channel>thoughtful musing"]), + vec![reasoning("thoughtful musing")] + ); + } + + #[test] + fn a_label_split_across_chunks_never_leaks_and_never_eats_content() { + // The undecided window: the buffer ends INSIDE `thought\n`, so the opener + // cannot be classified yet and must be held rather than released as text. + assert_eq!( + events( + &weather_tools(), + &["<|channel>", "thou", "ght\nheld"] + ), + vec![reasoning("held")] + ); + assert_eq!( + events( + &weather_tools(), + &["<|chan", "nel>thoug", "htful musing"] + ), + vec![reasoning("thoughtful musing")] + ); + } + + #[test] + fn markers_split_across_chunks_never_leak() { + // Every marker is cut in half at a chunk boundary, including the `<|"|>` + // string delimiter and the asymmetric end marker. + let out = events( + &weather_tools(), + &[ + "<|chan", + "nel>thought\nago: <|tool", + "_call>call:get_weather{city:<|", + "\"|>Par", + "is<|\"|", + ">}done", + ], + ); + assert_eq!( + out, + vec![ + reasoning("a"), + text("go: "), + call("get_weather", serde_json::json!({"city": "Paris"})), + text("done"), + ] + ); + } + + #[test] + fn unterminated_reasoning_is_promoted_at_finish() { + // 10.d: not dropped, and not leaked as visible text. + let out = events( + &weather_tools(), + &["<|channel>thought\nthinking but stream ends"], + ); + assert_eq!(out, vec![reasoning("thinking but stream ends")]); + } + + #[test] + fn an_argument_value_may_contain_the_block_close_marker() { + // 7.b / I7 — the case that made gemma4 Tier C. `` inside a + // `<|"|>` string is DATA, so a plain `find` of the end marker would cut + // the value here. The invoke scan resolves the real end instead. + let out = events( + &[tool("run", "cmd")], + &["<|tool_call>call:run{cmd:<|\"|>git log } --oneline<|\"|>}"], + ); + assert_eq!( + out, + vec![call( + "run", + serde_json::json!({"cmd": "git log } --oneline"}) + )] + ); + } + + #[test] + fn reasoning_marker_inside_an_argument_is_data() { + // 12.a / I7: once a tool block is open, `<|channel>` is a value, not a + // control token, and must survive byte-exact — role label and all. + let out = events( + &[tool("log", "note")], + &[ + "<|tool_call>call:log{note:<|\"|><|channel>thought\nreconsider<|\"|>}", + ], + ); + assert_eq!( + out, + vec![call( + "log", + serde_json::json!({"note": "<|channel>thought\nreconsider"}) + )] + ); + } + + #[test] + fn reasoning_marker_inside_an_argument_is_data_at_every_split() { + // 12.a / I7 at every chunk boundary: once a tool block is open, a + // `<|channel>` marker inside the argument string is data, byte-exact, + // regardless of where the stream happens to be cut. + let input = "<|tool_call>call:log{note:<|\"|><|channel>thought\nreconsider<|\"|>}"; + let expected = vec![call( + "log", + serde_json::json!({"note": "<|channel>thought\nreconsider"}), + )]; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + assert_eq!( + events(&[tool("log", "note")], &[&input[..split], &input[split..]]), + expected, + "split={split}" + ); + } + } + + #[test] + fn tool_call_inside_reasoning_is_extracted_and_splits_the_thought() { + // 12.b: tool structure dominates reasoning. A call the model emits inside + // a thought is still a real call, so it surfaces as its own event and the + // thought splits around it. Burying it would both drop the call and leak + // `<|tool_call>` markup into the reasoning payload (`I3`). + let out = events( + &weather_tools(), + &[ + "<|channel>thought\nI should check. <|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>} now answerDone.", + ], + ); + assert_eq!( + out, + vec![ + reasoning("I should check. "), + call("get_weather", serde_json::json!({"city": "Paris"})), + reasoning(" now answer"), + text("Done."), + ] + ); + } + + #[test] + fn tool_call_inside_reasoning_is_extracted_at_every_split() { + // 12.b at every chunk boundary: a call emitted inside a thought is + // extracted as its own event and the thought splits around it, + // regardless of where the stream happens to be cut. + let input = "<|channel>thought\nI should check. <|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>} now answerDone."; + let expected = vec![ + reasoning("I should check. "), + call("get_weather", serde_json::json!({"city": "Paris"})), + reasoning(" now answer"), + text("Done."), + ]; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + assert_eq!( + events(&weather_tools(), &[&input[..split], &input[split..]]), + expected, + "split={split}" + ); + } + } + + #[test] + fn narration_after_a_call_survives() { + // The block IS the invoke, so closing the call must close the block. A + // block left open would suppress every later text run as markup. + let out = events( + &weather_tools(), + &["<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>} Let me know."], + ); + assert_eq!( + out, + vec![ + call("get_weather", serde_json::json!({"city": "Paris"})), + text(" Let me know."), + ] + ); + } + + #[test] + fn orphan_close_is_stripped_not_leaked() { + // 5.c / I3: a `` with nothing open is malformed markup. + let out = events(&weather_tools(), &["I will check that. "]); + assert_eq!(out, vec![text("I will check that. ")]); + } + + #[test] + fn a_duplicate_channel_opener_inside_a_thought_is_stripped_with_its_label() { + // I3: a second `<|channel>` while one is already open is a duplicate + // opener, not content — and its role label goes with it, or `thought\n` + // lands in the reasoning payload. + let out = events( + &weather_tools(), + &["<|channel>thought\na<|channel>thought\nbtail"], + ); + assert_eq!(out, vec![reasoning("ab"), text("tail")]); + } + + #[test] + fn truncated_tool_call_at_eof_keeps_preceding_output() { + // 5.a / P2: drop the unrecoverable partial call, no error, no leaked markup. + let out = events( + &weather_tools(), + &[ + "<|channel>thought\nokChecking. <|tool_call>call:get_weather{city:<|\"|>Par", + ], + ); + assert_eq!(out, vec![reasoning("ok"), text("Checking. ")]); + } + + #[test] + fn truncated_tool_call_at_eof_keeps_preceding_output_at_every_split() { + // 5.a / P2 at every chunk boundary: the unrecoverable partial call must + // drop cleanly and the preceding reasoning/text must survive, no matter + // where the stream happens to be cut. + let input = + "<|channel>thought\nokChecking. <|tool_call>call:get_weather{city:<|\"|>Par"; + let expected = vec![reasoning("ok"), text("Checking. ")]; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + assert_eq!( + events(&weather_tools(), &[&input[..split], &input[split..]]), + expected, + "split={split}" + ); + } + } + + #[test] + fn a_complete_body_missing_its_close_marker_is_recovered_at_finish() { + // 5.b: the body balanced; only the close never streamed. v1 parity says + // recover it rather than drop it. + let out = events( + &weather_tools(), + &["<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>}"], + ); + assert_eq!( + out, + vec![call("get_weather", serde_json::json!({"city": "Paris"}))] + ); + } + + #[test] + fn a_complete_body_missing_its_close_marker_is_recovered_at_every_split() { + // 5.b at every chunk boundary: the body balances but the close marker + // never streams, so `finish` must recover it regardless of split point. + let input = "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>}"; + let expected = vec![call("get_weather", serde_json::json!({"city": "Paris"}))]; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + assert_eq!( + events(&weather_tools(), &[&input[..split], &input[split..]]), + expected, + "split={split}" + ); + } + } + + #[test] + fn later_balanced_call_with_trailing_whitespace_is_recovered_at_eof() { + let input = concat!( + "<|tool_call>call:broken{note:<|\"|>unterminated", + "<|\"|>}<|tool_call>call:get_weather{city:<|\"|>NYC<|\"|>} \n\t" + ); + let expected = vec![ + call("get_weather", serde_json::json!({"city": "NYC"})), + text(" \n\t"), + ]; + for chunk_size in [1, 4, 16] { + let chunks = input + .as_bytes() + .chunks(chunk_size) + .map(|chunk| std::str::from_utf8(chunk).expect("ASCII test input")) + .collect::>(); + assert_eq!( + events(&weather_tools(), &chunks), + expected, + "chunk_size={chunk_size}" + ); + } + } + + #[test] + fn concatenated_calls_stay_separate_events() { + // Gemma 4 concatenates calls with NO separator between them. + let out = events( + &[tool("f", "x"), tool("g", "y")], + &[ + "<|tool_call>call:f{x:<|\"|>1<|\"|>}<|tool_call>call:g{y:<|\"|>2<|\"|>}", + ], + ); + assert_eq!( + out, + vec![ + call("f", serde_json::json!({"x": "1"})), + call("g", serde_json::json!({"y": "2"})), + ] + ); + } + + #[test] + fn a_bare_call_without_its_opener_is_recovered_as_a_call() { + // Gemma 4's missing-start recovery: the body is a call, not prose, so it + // must not leak as visible text. Prose that merely contains the WORD + // `call:` still reaches the user untouched. + assert_eq!( + events( + &weather_tools(), + &["I will check. call:get_weather{city:<|\"|>Paris<|\"|>}"], + ), + vec![ + text("I will check. "), + call("get_weather", serde_json::json!({"city": "Paris"})), + ] + ); + assert_eq!( + events(&weather_tools(), &["I will call: you tomorrow."]), + vec![text("I will call: you tomorrow.")] + ); + } + + #[test] + fn a_bare_call_without_its_opener_is_recovered_at_every_split() { + // Gemma 4's missing-start recovery at every chunk boundary: the prose + // prefix stays text and the bare call body recovers as a call, no matter + // where the stream happens to be cut. + let input = "I will check. call:get_weather{city:<|\"|>Paris<|\"|>}"; + let expected = vec![ + text("I will check. "), + call("get_weather", serde_json::json!({"city": "Paris"})), + ]; + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + assert_eq!( + events(&weather_tools(), &[&input[..split], &input[split..]]), + expected, + "split={split}" + ); + } + } + + #[test] + fn malformed_wrapper_recovery_is_chunk_invariant() { + let cases = [ + ( + "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>} trailing ", + Vec::new(), + ), + ( + "for example, call:get_weather{city:<|\"|>Paris<|\"|>}", + vec![text("for example, ")], + ), + ]; + for (input, expected) in cases { + for split in input + .char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(input.len())) + { + assert_eq!( + events(&weather_tools(), &[&input[..split], &input[split..]]), + expected, + "input={input:?} split={split}" + ); + } + } + } + + #[test] + fn batch_and_stream_assemble_identically() { + // I6, at the parser level: `parse_complete` routes through the same + // push/finish lifecycle, so parity is structural. + let input = "<|channel>thought\naHere you go: <|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>}<|channel>thought\nbDone."; + let batch = gemma4_unified(&weather_tools()) + .parse_complete(input) + .expect("parse_complete"); + assert_eq!(batch, events(&weather_tools(), &[input])); + assert_eq!(batch.len(), 5); + } + + #[test] + fn reasoning_prefill_classifies_leading_text_as_reasoning() { + // 40.*: the prompt opened the channel AND wrote the role label, so the + // stream begins mid-thought with neither marker nor label. + let out = configured_events( + &weather_tools(), + UnifiedParserStartingState::Reasoning, + UnifiedToolOutputMode::Native, + &["hiddenvisible"], + ); + assert_eq!(out, vec![reasoning("hidden"), text("visible")]); + } + + #[test] + fn reasoning_prefill_consumes_a_redundant_opener_with_its_label() { + // 41.a: the backend re-emits the opener the prompt already wrote. Exactly + // one echo is consumed — including `thought\n`, which is part of it. + let out = configured_events( + &weather_tools(), + UnifiedParserStartingState::Reasoning, + UnifiedToolOutputMode::Native, + &[ + "<|channel>thought\nchecking weather", + "<|tool_call>call:get_weather{city:<|\"|>London<|\"|>}", + ], + ); + assert_eq!( + out, + vec![ + reasoning("checking weather"), + call("get_weather", serde_json::json!({"city": "London"})), + ] + ); + } + + #[test] + fn response_prefill_does_not_interpret_channel_markers() { + // 50.d — the only case where `starting_state=Response` is observable: the prompt + // opened VISIBLE content, so this stream has no reasoning channel and the + // channel markers are ordinary characters the user must see. + let out = configured_events( + &weather_tools(), + UnifiedParserStartingState::Response, + UnifiedToolOutputMode::Native, + &["<|channel>thought\nliteral then a call"], + ); + assert_eq!( + out, + vec![text("<|channel>thought\nliteral then a call")] + ); + } + + #[test] + fn guided_json_strips_the_channel_envelope_around_the_payload() { + // 40.b: guided decoding is a BACKEND feature, so the payload is bare JSON + // in every family — but the reasoning envelope around it is still the + // family's own markers, which is all `GuidedState` needs. + let out = configured_events( + &weather_tools(), + UnifiedParserStartingState::Reasoning, + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + &[ + "checking weather[{\"name\": \"get_weather\", ", + "\"arguments\": {\"city\": \"Paris\"}}]", + ], + ); + assert_eq!( + out, + vec![ + reasoning("checking weather"), + call("get_weather", serde_json::json!({"city": "Paris"})), + ] + ); + } + + #[test] + fn guided_json_named_choice_parses_a_bare_arguments_object() { + // 30.a: a named choice constrains output to that tool's arguments alone. + let out = configured_events( + &weather_tools(), + UnifiedParserStartingState::None, + UnifiedToolOutputMode::GuidedJson { + named_tool: Some("get_weather".to_string()), + }, + &["{\"city\": ", "\"Paris\"}"], + ); + assert_eq!( + out, + vec![call("get_weather", serde_json::json!({"city": "Paris"}))] + ); + } + + #[test] + fn guided_json_recovers_a_payload_that_is_not_a_call_as_text() { + // 31.a / P2: no `name`, so there is nothing to dispatch. Surface the bytes + // rather than dropping them or erroring. + let out = configured_events( + &weather_tools(), + UnifiedParserStartingState::None, + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + &["{\"unexpected\": \"shape\"}"], + ); + assert_eq!(out, vec![text("{\"unexpected\": \"shape\"}")]); + } + + /// gemma4's opener carries a `thought\n` ROLE LABEL. It is grammar, not + /// thought, so it must never surface as reasoning — in EITHER request mode, + /// at ANY chunk boundary. + /// + /// Enumerated from the property, not from a fix: every char-boundary split + /// of a labelled opener, crossed with native and guided. Sampling splits is + /// what let this defect reach the corpus in the first place. + /// + /// Negative control: make the guided drain consume `start.len()` instead of + /// `reasoning_opener_len(..)` and every guided row here fails with + /// `reasoning("thought\nchecking")`. + #[test] + fn a_role_label_is_never_thought_in_either_request_mode() { + let tools = weather_tools(); + let cases = [ + ( + "<|channel>thought\nchecking<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>}", + UnifiedToolOutputMode::Native, + ), + ( + "<|channel>thought\nchecking[{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}]", + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + ), + ]; + for (input, mode) in cases { + for split in 1..input.len() { + if !input.is_char_boundary(split) { + continue; + } + let (head, tail) = input.split_at(split); + let got = configured_events( + &tools, + UnifiedParserStartingState::None, + mode.clone(), + &[head, tail], + ); + let reasoning: String = got + .iter() + .filter_map(|e| match e { + UnifiedEvent::Reasoning { text } => Some(text.as_str()), + _ => None, + }) + .collect(); + assert_eq!( + reasoning, "checking", + "{mode:?} split at {split}: role label or thought text wrong; got {got:?}" + ); + } + } + } +} diff --git a/parsers/v2/src/unified/mod.rs b/parsers/v2/src/unified/mod.rs index b97fa0937..c9abb7375 100644 --- a/parsers/v2/src/unified/mod.rs +++ b/parsers/v2/src/unified/mod.rs @@ -48,6 +48,7 @@ //! adopted without a translation layer. Where it diverges from the peer shape, the //! divergence is stated at the item. +pub mod gemma4; mod guided_cursor; pub mod kimi_k2; pub mod muse_glimmer; @@ -60,8 +61,8 @@ pub use guided_cursor::{CommittedCall, GuidedJsonCursor}; use serde::{Deserialize, Serialize}; use crate::tool_calling::scan::{ - InvokeEmitter, InvokeScan, ReasoningSpec, WrappedBlockScanner, marker_prefix_suffix_len, - push_run, reasoning_opener_len, + InvokeBoundary, InvokeBoundaryFactory, InvokeEmitter, ReasoningSpec, WrappedBlockScanner, + marker_prefix_suffix_len, push_run, reasoning_opener_len, }; use crate::tool_calling::traits::{Result, Tool, ToolCallDelta, ToolParseResult}; @@ -684,11 +685,20 @@ impl ToolParseResult { /// the trait itself has no `create`. pub(crate) struct ScannerUnified { pub(crate) scanner: WrappedBlockScanner, + guided_prefix_policy: Option, } impl ScannerUnified { pub(crate) fn new(scanner: WrappedBlockScanner) -> Self { - Self { scanner } + Self { + scanner, + guided_prefix_policy: None, + } + } + + pub(crate) fn with_guided_prefix_policy(mut self, policy: GuidedPrefixPolicy) -> Self { + self.guided_prefix_policy = Some(policy); + self } } @@ -712,7 +722,8 @@ impl NativeUnified for ScannerUnified { control_markers: self.scanner.control_markers().to_vec(), invoke_start: self.scanner.invoke_start().to_string(), invoke_end: self.scanner.invoke_end().to_string(), - invoke_scan: self.scanner.invoke_scan(), + invoke_boundary_factory: self.scanner.invoke_boundary_factory(), + guided_prefix_policy: self.guided_prefix_policy, } } @@ -1321,7 +1332,31 @@ pub(crate) struct GuidedGrammar { pub(crate) control_markers: Vec, pub(crate) invoke_start: String, pub(crate) invoke_end: String, - pub(crate) invoke_scan: Option, + pub(crate) invoke_boundary_factory: Option, + pub(crate) guided_prefix_policy: Option, +} + +/// Family-owned recognition of syntax immediately before a guided JSON payload. +/// Native invocation scanning stays independent because this policy only affects +/// guided output framing. +pub(crate) type GuidedPrefixPolicy = fn(GuidedPrefixContext<'_>) -> GuidedPrefix; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum GuidedPrefix { + NoMatch, + Pending, + Match, + Strip, +} + +/// Context supplied to a family policy without exposing guided parser state. +#[derive(Clone, Copy)] +pub(crate) struct GuidedPrefixContext<'a> { + pub(crate) text: &'a str, + pub(crate) at: usize, + pub(crate) outside_reasoning: bool, + pub(crate) payload_is_empty: bool, + pub(crate) followed_by_competing_marker: bool, } struct GuidedState { @@ -1352,6 +1387,12 @@ struct GuidedState { /// both the inside-a-thought and outside-a-thought scopes. Assembling it /// per-site from openers and orphans is how those four uses drifted apart. grammar: GuidedGrammar, + /// Guided decoding owns a separate request-local boundary from native parsing. + invoke_boundary: Option>, + /// Candidate currently bound to `invoke_boundary`. Guided drains can revisit + /// retained bytes as reasoning/control state changes; this keeps the boundary + /// append-aware instead of starting its scan at byte zero each time. + invoke_candidate: String, named_tool: Option, invalid_payload: InvalidGuidedPayloadPolicy, /// Response starting_state disables reasoning markers, but tool control markers @@ -1537,6 +1578,28 @@ fn control_marker_len_at( } } +/// Hold back a guided prefix only when it can still occupy the trailing invoke-marker suffix. +fn guided_pending_suffix_len( + input: &str, + invoke_start: &str, + policy: F, + context: GuidedPrefixContext<'_>, +) -> usize +where + F: Fn(GuidedPrefixContext<'_>) -> GuidedPrefix, +{ + input + .char_indices() + .rev() + .take_while(|(at, _)| input.len() - *at <= invoke_start.len()) + .filter_map(|(at, _)| { + (policy(GuidedPrefixContext { at, ..context }) == GuidedPrefix::Pending) + .then_some(input.len() - at) + }) + .max() + .unwrap_or(0) +} + /// Trailing bytes the guided drain must retain across a chunk boundary. /// /// Two reasons to hold back, and missing either one loses the payload: @@ -1544,13 +1607,15 @@ fn control_marker_len_at( /// prefix-form marker still waiting for its terminator (``). /// The second is not a partial marker, so the prefix scan does not see it, and it /// was flushed into the payload buffer where it broke the parse. +#[allow(clippy::too_many_arguments)] fn guided_holdback_len( input: &str, reasoning_markers: &[&str], control: &[String], invoke_end: &str, + invoke_start: &str, start_label: Option<(&str, &str)>, - invoke_control: Option<(&str, InvokeScan)>, + guided_prefix_policy: Option, flush: bool, ) -> usize { if flush { @@ -1605,30 +1670,45 @@ fn guided_holdback_len( (rest.len() < label.len() && label.starts_with(rest)).then_some(input.len() - at) }) .unwrap_or(0); - let pending_invoke = invoke_control - .map(|(start, scan)| { - let partial = (scan.holdback)(input); - let body = input - .match_indices(start) + let context = GuidedPrefixContext { + text: input, + at: 0, + outside_reasoning: true, + payload_is_empty: true, + followed_by_competing_marker: false, + }; + let guided = guided_prefix_policy + .map(|policy| guided_pending_suffix_len(input, invoke_start, policy, context)) + .unwrap_or(0); + // A lexical prefix directly before an incomplete reasoning marker is undecided + // until the marker finishes. Releasing it first makes a whole-input parse strip + // malformed `call:` syntax while a split parse exposes it as visible text. + let before_reasoning_marker = guided_prefix_policy + .map(|policy| { + input + .match_indices(invoke_start) .filter_map(|(at, _)| { - let suffix = &input[at..]; - // `flush: false` here always makes `tool_index` inert (Kimi's - // EOF-only recovery gate can only fire when `flush` is true) -- - // 0 is not a claim about which call this is, just the value - // that keeps this holdback-length probe's result identical to - // before `tool_index` existed. - ((scan.opens)(input, at) && (scan.end)(suffix, false, 0).is_none()) + let tail = &input[at + invoke_start.len()..]; + let marker_tail = + marker_prefix_suffix_len(tail, reasoning_markers.iter().copied()); + (marker_tail == tail.len() + && marker_tail > 0 + && policy(GuidedPrefixContext { + at, + followed_by_competing_marker: true, + ..context + }) == GuidedPrefix::Strip) .then_some(input.len() - at) }) .max() - .unwrap_or(0); - partial.max(body) + .unwrap_or(0) }) .unwrap_or(0); split .max(pending_prefix_form) .max(pending_label) - .max(pending_invoke) + .max(guided) + .max(before_reasoning_marker) } /// Whether a buffered guided run has opened a JSON value. Anything before the @@ -1737,10 +1817,15 @@ impl GuidedState { Some(name) => GuidedJsonCursor::named(name.clone()), None => GuidedJsonCursor::new(), }; + let invoke_boundary = grammar + .invoke_boundary_factory + .map(InvokeBoundaryFactory::create); Self { stripped_markup: false, reasoning, grammar, + invoke_boundary, + invoke_candidate: String::new(), named_tool, invalid_payload, reasoning_enabled: starting_state != UnifiedParserStartingState::Response, @@ -1843,6 +1928,10 @@ impl GuidedState { self.payload_emitted = false; self.post_payload_text_started = false; self.cursor.reset(); + if let Some(boundary) = self.invoke_boundary.as_mut() { + boundary.reset(); + } + self.invoke_candidate.clear(); recovered } @@ -2028,6 +2117,43 @@ impl GuidedState { } } + fn invoke_end_append(&mut self, candidate: &str, flush: bool) -> Option { + let boundary = self.invoke_boundary.as_mut()?; + let append = if let Some(append) = candidate.strip_prefix(&self.invoke_candidate) { + append + } else { + boundary.reset(); + self.invoke_candidate.clear(); + candidate + }; + self.invoke_candidate.push_str(append); + boundary.end_append(candidate, append, flush, 0) + } + + fn reset_invoke_candidate(&mut self) { + if let Some(boundary) = self.invoke_boundary.as_mut() { + boundary.reset(); + } + self.invoke_candidate.clear(); + } + + /// Return the bytes owned by the append-aware native-envelope candidate. + /// The candidate is advanced by `control_marker_at`; holdback only observes + /// that state, so a one-byte stream never restarts the grammar scan. + fn invoke_holdback_len(&self) -> usize { + let native = self + .invoke_boundary + .as_ref() + .map(|boundary| boundary.holdback(&self.input)) + .unwrap_or(0); + let candidate = if self.invoke_candidate.is_empty() { + 0 + } else { + self.invoke_candidate.len() + }; + native.max(candidate) + } + /// Whether the bytes at `from` reach the guided payload through nothing but /// recognized control markup. /// @@ -2037,15 +2163,16 @@ impl GuidedState { /// `None` -- nothing has arrived yet and neither reading is safe; the caller must /// hold the bytes instead of committing to one, or the same input parses /// differently depending on where the chunk boundaries fell (`I6`). - fn leads_into_payload(&self, from: usize, flush: bool) -> Option { + fn leads_into_payload(&mut self, from: usize, flush: bool) -> Option { + let input = std::mem::take(&mut self.input); let mut at = from; - loop { - let rest = &self.input[at..]; + let result = loop { + let rest = &input[at..]; if json_payload_started(rest) { - return Some(true); + break Some(true); } if rest.trim().is_empty() { - return flush.then_some(false); + break flush.then_some(false); } // Step over further control markup sitting between the recovery point and // the payload -- a block opener wrapping the payload is exactly this shape. @@ -2059,19 +2186,15 @@ impl GuidedState { }); match skip { Some((_, len)) if len > 0 => at += len, - _ => return Some(false), + _ => break Some(false), } - } - } - - fn invoke_control(&self) -> Option<(&str, InvokeScan)> { - self.grammar - .invoke_scan - .map(|scan| (self.grammar.invoke_start.as_str(), scan)) + }; + self.input = input; + result } fn control_marker_at( - &self, + &mut self, haystack: &str, limit: Option, competing: &[&str], @@ -2085,15 +2208,59 @@ impl GuidedState { competing, flush, ); - let Some(scan) = self.grammar.invoke_scan else { + if self.invoke_boundary.is_none() { return regular; - }; + } let mut cursor = 0; + let guided_prefix_at_payload_boundary = + self.mode == GuidedMode::OutsideReasoning && self.json.trim().is_empty(); while let Some(relative) = haystack[cursor..].find(&self.grammar.invoke_start) { let at = cursor + relative; let suffix = &haystack[at..]; - if (scan.opens)(haystack, at) { + if limit.is_some_and(|limit| at >= limit) { + break; + } + let prefix = self.grammar.guided_prefix_policy.map(|policy| { + policy(GuidedPrefixContext { + text: haystack, + at, + outside_reasoning: self.mode == GuidedMode::OutsideReasoning, + payload_is_empty: self.json.trim().is_empty(), + followed_by_competing_marker: competing.iter().any(|marker| { + suffix[self.grammar.invoke_start.len()..].starts_with(marker) + }), + }) + }); + match prefix { + Some(GuidedPrefix::Match) => { + if !guided_prefix_at_payload_boundary { + cursor = at + self.grammar.invoke_start.len(); + continue; + } + return regular + .filter(|(regular_at, _)| *regular_at < at) + .or(Some((at, self.grammar.invoke_start.len()))); + } + Some(GuidedPrefix::Pending) if !flush => { + if !guided_prefix_at_payload_boundary { + cursor = at + self.grammar.invoke_start.len(); + continue; + } + return regular.filter(|(regular_at, _)| *regular_at < at); + } + Some(GuidedPrefix::Strip) => { + return regular + .filter(|(regular_at, _)| *regular_at < at) + .or(Some((at, self.grammar.invoke_start.len()))); + } + Some(GuidedPrefix::Pending | GuidedPrefix::NoMatch) | None => {} + } + if self + .invoke_boundary + .as_ref() + .is_some_and(|boundary| boundary.opens(haystack, at)) + { // `tool_index: 0` is provably safe here, not merely convenient: // this loop can NEVER walk past a first `invoke_start` match to // examine a second one for a family whose `opens` hook always @@ -2115,9 +2282,10 @@ impl GuidedState { .any(|marker| haystack[*boundary..].starts_with(marker)) }); let local_flush = flush || competing_boundary.is_some(); - if let Some(len) = (scan.end)(suffix, local_flush, 0) + if let Some(len) = self.invoke_end_append(suffix, local_flush) && competing_boundary.is_none_or(|boundary| at + len <= boundary) { + self.reset_invoke_candidate(); return regular .filter(|(regular_at, _)| *regular_at < at) .or(Some((at, len))); @@ -2127,6 +2295,7 @@ impl GuidedState { // Strip only through that boundary so the reasoning closer // and guided JSON after it are scanned independently. if let Some(boundary) = competing_boundary { + self.reset_invoke_candidate(); return regular .filter(|(regular_at, _)| *regular_at < at) .or(Some((at, boundary - at))); @@ -2135,15 +2304,22 @@ impl GuidedState { // unrecoverable partial call (P2), not visible recovery text. // No future bytes can establish a narrower safe boundary. if flush { + self.reset_invoke_candidate(); return regular .filter(|(regular_at, _)| *regular_at < at) .or(Some((at, suffix.len()))); } return regular.filter(|(regular_at, _)| *regular_at < at); } - if !flush && (scan.holdback)(suffix) == suffix.len() { + if !flush + && self + .invoke_boundary + .as_ref() + .is_some_and(|boundary| boundary.holdback(suffix) == suffix.len()) + { return regular.filter(|(regular_at, _)| *regular_at < at); } + self.reset_invoke_candidate(); cursor = at + self.grammar.invoke_start.len(); } regular @@ -2222,18 +2398,21 @@ impl GuidedState { }) .flatten() .map(|(at, _)| at); - let stray_close = self - .control_marker_at( - &self.input, - // Nor past the start of the payload itself. - self.input.find(['{', '[']), - // A thought marker ahead also ends the header: a stray - // `` from `` and - // swallow the thought — that put private reasoning in the - // user's answer. - &reasoning_markers, - flush, - ) + let payload_start = self.input.find(['{', '[']); + let input = std::mem::take(&mut self.input); + let marker = self.control_marker_at( + &input, + // Nor past the start of the payload itself. + payload_start, + // A thought marker ahead also ends the header: a stray + // `` from `` and + // swallow the thought — that put private reasoning in the + // user's answer. + &reasoning_markers, + flush, + ); + self.input = input; + let stray_close = marker .into_iter() .chain( self.reasoning_enabled @@ -2324,6 +2503,9 @@ impl GuidedState { self.note_consumed(at, close_len); self.note_content_transition(at, close_len); self.input.drain(..at + close_len); + // A competing marker was stripped while the native-looking + // candidate stayed buffered. It is not a candidate discard, + // so keep its boundary progress for the next drain. continue; } @@ -2339,15 +2521,22 @@ impl GuidedState { } else { &[] }; + let start_label = self + .start_label() + .map(|(start, label)| (start.to_owned(), label.to_owned())); guided_holdback_len( &self.input, held, &self.grammar.control_markers, &self.grammar.invoke_end, - self.start_label(), - self.invoke_control(), + &self.grammar.invoke_start, + start_label + .as_ref() + .map(|(start, label)| (start.as_str(), label.as_str())), + self.grammar.guided_prefix_policy, flush, ) + .max(self.invoke_holdback_len()) .max(if self.reasoning_enabled { self.reasoning.holdback(&self.input, self.channel_state()) } else { @@ -2490,15 +2679,17 @@ impl GuidedState { let reopen = self .reasoning .find_open(&self.input, flush, self.channel_state()); - let interrupt = self - .control_marker_at( - &self.input, - // A narrated invoke lives INSIDE this thought, so its - // terminator cannot be past the span's closer. - ends.map(|(at, _)| at), - &close_markers, - flush, - ) + let input = std::mem::take(&mut self.input); + let marker = self.control_marker_at( + &input, + // A narrated invoke lives INSIDE this thought, so its + // terminator cannot be past the span's closer. + ends.map(|(at, _)| at), + &close_markers, + flush, + ); + self.input = input; + let interrupt = marker .into_iter() .chain( self.reasoning @@ -2585,15 +2776,36 @@ impl GuidedState { let keep = if flush { 0 } else { + // Same `invoke_scan` awareness the `stray` lookup above + // already applies to this exact channel: a COMPLETE + // `invoke_start` marker sitting at the tail of buffered + // reasoning text, with its own `scan.end` still Pending + // (e.g. `<|tool_call_begin|>` with no `functions.NAME` + // after it yet), used to fall straight through this `keep` + // computation because it hardcoded `None` here instead of + // `self.invoke_control()` -- unlike the `OutsideReasoning` + // arm's own holdback call, which already passes it. That + // flushed the marker out as literal reasoning text one + // push early, before `control_marker_at` ever got a + // chance to classify it once the rest of the header + // streamed in on a later push, and no chunking split it + // could still be recovered. + let start_label = self + .start_label() + .map(|(start, label)| (start.to_owned(), label.to_owned())); guided_holdback_len( &self.input, &reasoning_markers, &self.grammar.control_markers, &self.grammar.invoke_end, - self.start_label(), - self.invoke_control(), + &self.grammar.invoke_start, + start_label + .as_ref() + .map(|(start, label)| (start.as_str(), label.as_str())), + self.grammar.guided_prefix_policy, flush, ) + .max(self.invoke_holdback_len()) .max(self.reasoning.holdback(&self.input, self.channel_state())) .max(undecided_at.map_or(0, |at| self.input.len() - at)) }; @@ -3280,6 +3492,7 @@ macro_rules! unified_registry { } unified_registry! { + "gemma4" => gemma4::gemma4_unified, "qwen3" | "qwen3_coder" => qwen3::qwen3_unified, "muse_glimmer" => muse_glimmer::muse_glimmer_unified, "kimi_k2" => kimi_k2::kimi_k2_unified, @@ -3390,6 +3603,7 @@ impl UnifiedParser for DebugUnifiedParser { #[cfg(test)] mod tests { use super::*; + use std::cell::Cell; fn call(tool_index: usize, name: Option<&str>, arguments: &str) -> UnifiedParserEvent { UnifiedParserEvent::ToolCall(ToolCallDelta { @@ -3399,6 +3613,38 @@ mod tests { }) } + #[test] + fn guided_pending_probe_is_bounded_to_the_invoke_suffix() { + let input = "x".repeat(32 * 1024); + let probes = Cell::new(0); + + let policy = |_: GuidedPrefixContext<'_>| { + probes.set(probes.get() + 1); + GuidedPrefix::NoMatch + }; + assert_eq!( + guided_pending_suffix_len( + &input, + "call:", + policy, + GuidedPrefixContext { + text: &input, + at: 0, + outside_reasoning: true, + payload_is_empty: true, + followed_by_competing_marker: false, + }, + ), + 0 + ); + assert!( + probes.get() <= "call:".len(), + "guided holdback probed {} offsets for a {}-byte marker", + probes.get(), + "call:".len() + ); + } + #[test] fn guided_payload_boundary_accepts_multibyte_trailing_text() { let reasoning = GuidedReasoning::Pair(ReasoningSpec { @@ -3443,7 +3689,8 @@ mod tests { control_markers: vec!["".into(), "".into()], invoke_start: "".into(), - invoke_scan: None, + invoke_boundary_factory: None, + guided_prefix_policy: None, }, None, UnifiedParserStartingState::None, @@ -3645,6 +3892,58 @@ mod tests { assert_eq!(plain.reset(), debug.reset()); } + #[test] + fn debug_wrapper_matches_gemma_across_reset_and_guided_reuse() { + let tools = vec![Tool { + name: "get_weather".into(), + description: None, + parameters: serde_json::json!({"type": "object"}), + strict: None, + }]; + let mut plain = gemma4::gemma4_unified(&tools); + let mut debug = DebugUnifiedParser::wrap("gemma4", gemma4::gemma4_unified(&tools)); + let initial = UnifiedParserInit { + tool_output_mode: UnifiedToolOutputMode::GuidedJson { named_tool: None }, + invalid_guided_payload: InvalidGuidedPayloadPolicy::RecoverAsText, + ..UnifiedParserInit::default() + }; + + for parser in [&mut plain, &mut debug] { + parser + .initialize_request(initial.clone()) + .expect("initial guided request"); + assert!( + parser + .push("<|tool_call>call:get_weather{city:<|\"|>Par") + .expect("partial native-looking candidate") + .is_empty() + ); + } + assert_eq!(plain.reset(), debug.reset()); + + let reuse = UnifiedParserInit { + tool_output_mode: UnifiedToolOutputMode::GuidedJson { + named_tool: Some("get_weather".into()), + }, + invalid_guided_payload: InvalidGuidedPayloadPolicy::StreamBestEffort, + ..UnifiedParserInit::default() + }; + for parser in [&mut plain, &mut debug] { + parser + .initialize_request(reuse.clone()) + .expect("reused guided request"); + } + assert_eq!( + plain.push(r#"{"city":"Paris"}"#).expect("guided JSON"), + debug.push(r#"{"city":"Paris"}"#).expect("guided JSON"), + "the wrapper must preserve Gemma's pre-finish guided commit" + ); + assert_eq!( + plain.finish().expect("finish").events, + debug.finish().expect("finish").events + ); + } + #[test] fn unified_event_matches_the_golden_corpus_schema() { let yaml = "- {kind: reasoning, text: \"a\"}\n\ diff --git a/parsers/v2/src/unified/muse_glimmer.rs b/parsers/v2/src/unified/muse_glimmer.rs index 3237cbd02..febb55ae5 100644 --- a/parsers/v2/src/unified/muse_glimmer.rs +++ b/parsers/v2/src/unified/muse_glimmer.rs @@ -84,7 +84,8 @@ impl NativeUnified for MuseChannelScanner { // The marker-only rules are enough here: an ATEM invoke is delimited by a // literal opener and closer, with no grammar-aware location rule of the // kind gemma4's value wrapping needs. - invoke_scan: None, + invoke_boundary_factory: None, + guided_prefix_policy: None, } } diff --git a/parsers/v2/tests/gemma4_guided.rs b/parsers/v2/tests/gemma4_guided.rs new file mode 100644 index 000000000..bbb6ae43a --- /dev/null +++ b/parsers/v2/tests/gemma4_guided.rs @@ -0,0 +1,310 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gemma 4 guided-mode behavior through the public unified-parser factory. + +use dynamo_parsers_v2::{ + InvalidGuidedPayloadPolicy, Tool, UnifiedEvent, UnifiedParserExt, UnifiedParserInit, + UnifiedParserStartingState, UnifiedToolOutputMode, assemble, +}; + +fn weather_tools() -> Vec { + vec![Tool { + name: "get_weather".into(), + description: None, + parameters: serde_json::json!({ + "type": "object", + "properties": { "city": { "type": "string" } } + }), + strict: None, + }] +} + +fn assert_guided_at_every_split( + input: &str, + mode: UnifiedToolOutputMode, + expected: &[UnifiedEvent], +) { + let tools = weather_tools(); + let mut split_points = vec![None]; + split_points.extend( + (1..input.len()) + .filter(|&split| input.is_char_boundary(split)) + .map(Some), + ); + for split in split_points { + let mut parser = dynamo_parsers_v2::create_unified_parser_for_family("gemma4", &tools) + .expect("built-in Gemma 4 parser"); + parser + .initialize_request(UnifiedParserInit { + starting_state: UnifiedParserStartingState::None, + tool_output_mode: mode.clone(), + invalid_guided_payload: InvalidGuidedPayloadPolicy::RecoverAsText, + ..UnifiedParserInit::default() + }) + .expect("guided request"); + + let chunks = split.map_or_else(|| vec![input], |at| vec![&input[..at], &input[at..]]); + let mut deltas = Vec::new(); + for chunk in chunks { + deltas.extend(parser.push(chunk).expect("push")); + } + deltas.extend(parser.finish().expect("finish").events); + assert_eq!(assemble(&deltas), expected, "split {split:?}"); + } +} + +fn guided_init(named_tool: Option<&str>, policy: InvalidGuidedPayloadPolicy) -> UnifiedParserInit { + UnifiedParserInit { + starting_state: UnifiedParserStartingState::None, + tool_output_mode: UnifiedToolOutputMode::GuidedJson { + named_tool: named_tool.map(str::to_owned), + }, + invalid_guided_payload: policy, + ..UnifiedParserInit::default() + } +} + +/// Gemma's lexical `call:` prefix is only structural when the grammar-aware +/// scanner accepts a call body. Guided mode must not strip the same bytes from +/// ordinary visible prose, even when the prefix is split across pushes. This +/// stays a public-factory regression instead of joining the unified corpus: +/// the corpus generator cannot add one family-specific guided input without +/// creating irrelevant rows for every guided family. +#[test] +fn preserves_ordinary_call_prefix_at_every_split() { + let cases = [ + ( + "I will call: you tomorrow<|channel>thought\nchecking[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}]", + "I will call: you tomorrow", + "checking", + ), + ( + "prefix call:{\"x\":1}<|channel>thought\nwhy[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}]", + "prefix call:{\"x\":1}", + "why", + ), + ]; + for (input, visible, reasoning) in cases { + let expected = vec![ + UnifiedEvent::Text { + text: visible.into(), + }, + UnifiedEvent::Reasoning { + text: reasoning.into(), + }, + UnifiedEvent::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }, + ]; + assert_guided_at_every_split( + input, + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + &expected, + ); + } +} + +#[test] +fn consumes_valid_call_prefix_at_every_split() { + let expected = [UnifiedEvent::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }]; + let cases = [ + ( + "call:[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}]", + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + ), + ( + "call:{\"city\":\"Paris\"}", + UnifiedToolOutputMode::GuidedJson { + named_tool: Some("get_weather".into()), + }, + ), + ]; + for (input, mode) in cases { + assert_guided_at_every_split(input, mode, &expected); + } +} + +#[test] +fn strips_malformed_call_prefixes_without_losing_reasoning_at_every_split() { + let cases = [ + ( + "call:<|channel>thought\nsecret[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}]", + "secret", + ), + ( + "<|channel>thought\nI'll call call:get_weather[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}]", + "I'll call get_weather", + ), + ]; + for (input, reasoning) in cases { + let expected = vec![ + UnifiedEvent::Reasoning { + text: reasoning.into(), + }, + UnifiedEvent::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }, + ]; + assert_guided_at_every_split( + input, + UnifiedToolOutputMode::GuidedJson { named_tool: None }, + &expected, + ); + } +} + +/// A public factory parser can abandon an incomplete native-looking candidate, +/// then serve a fresh guided request without carrying scanner or cursor state +/// across requests. +#[test] +fn reset_after_partial_native_candidate_reuses_the_public_factory_for_guided_json() { + let tools = weather_tools(); + let mut parser = dynamo_parsers_v2::create_unified_parser_for_family("gemma4", &tools) + .expect("built-in Gemma 4 parser"); + parser + .initialize_request(guided_init(None, InvalidGuidedPayloadPolicy::RecoverAsText)) + .expect("first guided request"); + + assert!( + parser + .push("<|tool_call>call:get_weather{city:<|\"|>Par") + .expect("partial native-looking candidate") + .is_empty() + ); + assert_eq!( + parser.reset(), + "call:get_weather{city:<|\"|>Par", + "the recognized native wrapper is control markup, while its incomplete body is recovered" + ); + + parser + .initialize_request(guided_init( + Some("get_weather"), + InvalidGuidedPayloadPolicy::RecoverAsText, + )) + .expect("fresh named guided request"); + let mut events = parser.push(r#"{"city":"Paris"}"#).expect("guided JSON"); + events.extend(parser.finish().expect("finish").events); + assert_eq!( + assemble(&events), + vec![UnifiedEvent::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }] + ); +} + +/// A rejected request setup must not poison a parser that is reset before a +/// valid guided request. This exercises the public lifecycle rather than the +/// generic router directly. +#[test] +fn rejected_guided_initialization_then_reset_allows_valid_initialization() { + let tools = weather_tools(); + let mut parser = dynamo_parsers_v2::create_unified_parser_for_family("gemma4", &tools) + .expect("built-in Gemma 4 parser"); + parser + .initialize_request(UnifiedParserInit::default()) + .expect("native request"); + parser.push("visible").expect("start native request"); + + assert!( + parser + .initialize_request(guided_init(None, InvalidGuidedPayloadPolicy::RecoverAsText)) + .is_err() + ); + assert_eq!(parser.reset(), ""); + + parser + .initialize_request(guided_init(None, InvalidGuidedPayloadPolicy::RecoverAsText)) + .expect("valid guided request after rejected setup"); + let mut events = parser + .push(r#"[{"name":"get_weather","arguments":{"city":"Paris"}}]"#) + .expect("guided JSON"); + events.extend(parser.finish().expect("finish").events); + assert_eq!( + assemble(&events), + vec![UnifiedEvent::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }] + ); +} + +/// Native-looking markup is suppressed immediately. Once a real reasoning +/// channel closes, it must be emitted on that push, and guided JSON must commit +/// before finish under the explicitly streaming policy. +#[test] +fn malformed_native_envelope_then_reasoning_and_guided_json_emit_at_their_crossings() { + let tools = weather_tools(); + let mut parser = dynamo_parsers_v2::create_unified_parser_for_family("gemma4", &tools) + .expect("built-in Gemma 4 parser"); + parser + .initialize_request(guided_init( + None, + InvalidGuidedPayloadPolicy::StreamBestEffort, + )) + .expect("streaming guided request"); + + assert!( + parser + .push("<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>}") + .expect("native-looking envelope") + .is_empty() + ); + assert_eq!( + parser + .push("<|channel>thought\nchecking") + .expect("completed reasoning"), + vec![dynamo_parsers_v2::UnifiedParserEvent::Reasoning( + "checking".into() + )], + "reasoning must not wait for guided JSON or finish" + ); + let json_events = parser + .push(r#"[{"name":"get_weather","arguments":{"city":"Paris"}}]"#) + .expect("completed guided JSON"); + assert!( + json_events.iter().any(|event| matches!( + event, + dynamo_parsers_v2::UnifiedParserEvent::ToolCall(call) + if call.name.as_deref() == Some("get_weather") + )), + "guided JSON must commit when its call crosses completion, before finish" + ); + assert!(parser.finish().expect("finish").events.is_empty()); +} + +/// Kimi does not install Gemma's optional `call:` policy. Its existing guided +/// JSON input must still dispatch normally, proving the generic router treats an +/// absent policy as the former no-op behavior. +#[test] +fn kimi_guided_dispatch_is_unchanged_without_a_prefix_policy() { + let tools = weather_tools(); + let mut parser = dynamo_parsers_v2::create_unified_parser_for_family("kimi_k2", &tools) + .expect("built-in Kimi K2 parser"); + parser + .initialize_request(UnifiedParserInit { + starting_state: UnifiedParserStartingState::None, + tool_output_mode: UnifiedToolOutputMode::GuidedJson { named_tool: None }, + invalid_guided_payload: InvalidGuidedPayloadPolicy::RecoverAsText, + ..UnifiedParserInit::default() + }) + .expect("guided request"); + + let input = r#"[{"name":"get_weather","arguments":{"city":"Paris"}}]"#; + let mut events = parser.push(input).expect("push"); + events.extend(parser.finish().expect("finish").events); + assert_eq!( + assemble(&events), + vec![UnifiedEvent::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }] + ); +} diff --git a/parsers/v2/tests/gemma4_native_recovery.rs b/parsers/v2/tests/gemma4_native_recovery.rs new file mode 100644 index 000000000..4f7f9cb25 --- /dev/null +++ b/parsers/v2/tests/gemma4_native_recovery.rs @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gemma 4 native-mode recovery through both public parser adapters. + +use dynamo_parsers_v2::tool_calling::create_tool_parser_for_family; +use dynamo_parsers_v2::{ + Tool, ToolParseResult, UnifiedEvent, UnifiedParserExt, UnifiedParserInit, assemble, +}; + +#[cfg(feature = "test-utils")] +use dynamo_parsers_v2::tool_calling::gemma4::{ + boundary_examined_bytes, reset_boundary_examined_bytes, +}; + +fn weather_tools() -> Vec { + vec![Tool { + name: "get_weather".into(), + description: None, + parameters: serde_json::json!({ + "type": "object", + "properties": { "city": { "type": "string" } } + }), + strict: None, + }] +} + +fn chunkings(input: &str) -> Vec> { + let mut chunks = vec![vec![input]]; + chunks.extend( + (1..input.len()) + .filter(|&at| input.is_char_boundary(at)) + .map(|at| vec![&input[..at], &input[at..]]), + ); + chunks +} + +fn assert_tool_only_at_every_split(input: &str, expected_text: &str) { + let tools = weather_tools(); + for chunks in chunkings(input) { + let mut parser = create_tool_parser_for_family("gemma4", &tools).expect("Gemma parser"); + let mut output = ToolParseResult::default(); + for chunk in &chunks { + output.append(parser.push(chunk).expect("push")); + } + output.append(parser.finish().expect("finish")); + let output = output.coalesce_calls(); + assert_eq!(output.normal_text, expected_text, "chunks={chunks:?}"); + assert_eq!(output.calls.len(), 1, "chunks={chunks:?}"); + assert_eq!(output.calls[0].name.as_deref(), Some("get_weather")); + assert_eq!(output.calls[0].arguments, r#"{"city":"NYC"}"#); + } +} + +fn assert_unified_at_every_split(input: &str, expected_text: &str) { + let tools = weather_tools(); + let expected = if expected_text.is_empty() { + vec![UnifiedEvent::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "NYC"}), + }] + } else { + vec![ + UnifiedEvent::Text { + text: expected_text.into(), + }, + UnifiedEvent::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "NYC"}), + }, + ] + }; + + for chunks in chunkings(input) { + let mut parser = dynamo_parsers_v2::create_unified_parser_for_family("gemma4", &tools) + .expect("Gemma unified parser"); + parser + .initialize_request(UnifiedParserInit::default()) + .expect("native request"); + let mut events = Vec::new(); + for chunk in &chunks { + events.extend(parser.push(chunk).expect("push")); + } + events.extend(parser.finish().expect("finish").events); + assert_eq!(assemble(&events), expected, "chunks={chunks:?}"); + } +} + +fn assert_both_adapters(input: &str, expected_text: &str) { + assert_tool_only_at_every_split(input, expected_text); + assert_unified_at_every_split(input, expected_text); +} + +fn assert_both_adapters_at_chunk_sizes(input: &str, chunk_sizes: &[usize], expected_calls: usize) { + let tools = weather_tools(); + for &chunk_size in chunk_sizes { + let chunks: Vec<_> = input + .as_bytes() + .chunks(chunk_size) + .map(|chunk| std::str::from_utf8(chunk).expect("ASCII Gemma fixture")) + .collect(); + + let mut tool_only = create_tool_parser_for_family("gemma4", &tools).expect("Gemma parser"); + let mut tool_output = ToolParseResult::default(); + for chunk in &chunks { + tool_output.append(tool_only.push(chunk).expect("tool-only push")); + } + tool_output.append(tool_only.finish().expect("tool-only finish")); + assert_eq!( + tool_output.coalesce_calls().calls.len(), + expected_calls, + "tool-only chunk_size={chunk_size}" + ); + + let mut unified = dynamo_parsers_v2::create_unified_parser_for_family("gemma4", &tools) + .expect("Gemma unified parser"); + unified + .initialize_request(UnifiedParserInit::default()) + .expect("native request"); + let mut events = Vec::new(); + for chunk in &chunks { + events.extend(unified.push(chunk).expect("unified push")); + } + events.extend(unified.finish().expect("unified finish").events); + let call_count = assemble(&events) + .into_iter() + .filter(|event| matches!(event, UnifiedEvent::ToolCall { .. })) + .count(); + assert_eq!( + call_count, expected_calls, + "unified chunk_size={chunk_size}" + ); + } +} + +#[test] +fn later_balanced_call_without_wrapper_close_recovers_at_eof_at_each_chunk_size() { + let malformed = "<|tool_call>call:broken{note:<|\"|>unterminated"; + let later_balanced = "<|\"|>}<|tool_call>call:get_weather{city:<|\"|>NYC<|\"|>}"; + let input = format!("{malformed}{later_balanced}"); + assert_both_adapters_at_chunk_sizes(&input, &[1, 4, 16], 1); +} + +#[test] +fn prose_call_prefix_does_not_capture_a_later_wrapped_call() { + let input = concat!( + "I will call: you tomorrow", + "<|tool_call>call:get_weather{city:<|\"|>NYC<|\"|>}", + ); + assert_both_adapters(input, "I will call: you tomorrow"); +} + +#[test] +fn malformed_block_resynchronizes_to_a_later_valid_block() { + let input = concat!( + "<|tool_call>call:broken{city:<|\"|>Paris<|\"|>}", + "<|tool_call>call:get_weather{city:<|\"|>NYC<|\"|>}", + ); + assert_both_adapters(input, ""); +} + +#[test] +fn incomplete_intermediate_block_does_not_hide_a_later_valid_block() { + let input = concat!( + "<|tool_call>call:broken{city:<|\"|>Paris<|\"|>}", + "<|tool_call>call:still_broken{nested:{", + "<|tool_call>call:get_weather{city:<|\"|>NYC<|\"|>}", + ); + assert_both_adapters(input, ""); +} + +#[test] +fn string_data_cannot_become_a_resynchronization_target() { + let input = concat!( + "<|tool_call>call:broken{note:<|\"|>", + "<|tool_call>call:get_weather{city:<|\"|>TRAP<|\"|>}", + "<|\"|>}", + "<|tool_call>call:get_weather{city:<|\"|>NYC<|\"|>}", + ); + assert_both_adapters(input, ""); +} + +#[test] +fn long_incremental_invokes_preserve_both_adapter_contracts() { + let value = "x".repeat(32 * 1024); + let valid = format!("<|tool_call>call:get_weather{{city:<|\"|>{value}<|\"|>}}"); + let incomplete = format!("<|tool_call>call:get_weather{{city:<|\"|>{value}"); + assert_both_adapters_at_chunk_sizes(&valid, &[4, 16], 1); + assert_both_adapters_at_chunk_sizes(&incomplete, &[4, 16], 0); +} + +#[test] +fn repeated_unmatched_wrappers_recover_the_later_complete_call() { + let malformed = "<|tool_call>call:broken{note:<|\"|>unterminated<|\"|>".repeat(256); + let input = + format!("{malformed}<|tool_call>call:get_weather{{city:<|\"|>NYC<|\"|>}}"); + assert_both_adapters_at_chunk_sizes(&input, &[4, 16], 1); +} + +/// This cannot be represented by the shared corpus: its assertion is a bound on +/// scanner work, not parser output. Exercise each exported adapter factory so a +/// future wiring change cannot bypass Gemma's request-local resynchronizer. +#[cfg(feature = "test-utils")] +#[test] +fn repeated_closers_in_an_unterminated_string_resynchronize_in_linear_time() { + for chunk_size in [1, 7, 64] { + let mut tool_only_scans = Vec::new(); + let mut unified_scans = Vec::new(); + + for repeats in [4 * 1024, 8 * 1024] { + let input = format!( + "<|tool_call>call:broken{{note:<|\"|>{}", + "".repeat(repeats) + ); + let chunks: Vec<_> = input + .as_bytes() + .chunks(chunk_size) + .map(|chunk| std::str::from_utf8(chunk).expect("ASCII Gemma fixture")) + .collect(); + + reset_boundary_examined_bytes(); + let mut tool_only = create_tool_parser_for_family("gemma4", &weather_tools()) + .expect("Gemma tool-only parser"); + let mut output = ToolParseResult::default(); + for chunk in &chunks { + output.append(tool_only.push(chunk).expect("tool-only push")); + } + output.append(tool_only.finish().expect("tool-only finish")); + assert!( + output.coalesce_calls().calls.is_empty(), + "unterminated string cannot recover a call at chunk_size={chunk_size}" + ); + tool_only_scans.push(boundary_examined_bytes()); + + reset_boundary_examined_bytes(); + let mut unified = + dynamo_parsers_v2::create_unified_parser_for_family("gemma4", &weather_tools()) + .expect("Gemma unified parser"); + unified + .initialize_request(UnifiedParserInit::default()) + .expect("native request"); + let mut events = Vec::new(); + for chunk in &chunks { + events.extend(unified.push(chunk).expect("unified push")); + } + events.extend(unified.finish().expect("unified finish").events); + assert!( + assemble(&events).is_empty(), + "unterminated string cannot recover unified events at chunk_size={chunk_size}" + ); + unified_scans.push(boundary_examined_bytes()); + } + + for (adapter, scans) in [("tool-only", tool_only_scans), ("unified", unified_scans)] { + let [at_n, at_2n] = scans.as_slice() else { + unreachable!("the N/2N measurement always has two inputs"); + }; + assert!( + *at_2n <= *at_n * 3, + "{adapter} rescanned superlinearly: N={at_n}, 2N={at_2n}, chunk_size={chunk_size}" + ); + } + } +}