Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions conformance/fixtures-manifest.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"snapshot": "20260827_110017",
"created_pt": "2026-08-27 11:00:17 America/Los_Angeles",
"snapshot": "20260828_105445",
"created_pt": "2026-08-28 10:54:45 America/Los_Angeles",
"crates": {
"dynamo-parsers": "8.2.0",
"dynamo-parsers-v2": "0.3.4"
Expand Down Expand Up @@ -230,6 +230,11 @@
"sha256": "ab6021d70826d91174a05a6bf143d7661502991440aace22983ba014b9475f4d",
"size": 13798
},
{
"path": "unified/dynamo_v2-0.3.4+pr166.tar.gz",
"sha256": "8d8971bcedb291930255359886cecb7e1cc73624385d8c574d0e239fc7f5835d",
"size": 13624
},
{
"path": "unified/dynamo_v2-0.3.4.tar.gz",
"sha256": "19b6f45dc869378a86af35a4a06395cbb6fb3f8976705ded2738405a536bca25",
Expand Down
3 changes: 3 additions & 0 deletions conformance/fixtures/unified/dynamo_v2-0.2.1.patch1.tar.gz
Git LFS file not shown
Git LFS file not shown
3 changes: 3 additions & 0 deletions conformance/fixtures/unified/dynamo_v2-0.3.2.patch1.tar.gz
Git LFS file not shown
3 changes: 3 additions & 0 deletions conformance/fixtures/unified/dynamo_v2-0.3.3+pr166.tar.gz
Git LFS file not shown
3 changes: 3 additions & 0 deletions conformance/fixtures/unified/dynamo_v2-0.3.4+pr166.tar.gz
Git LFS file not shown
Git LFS file not shown
55 changes: 26 additions & 29 deletions conformance/tests/common/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -236,48 +236,45 @@ pub fn fixture_name(path: &Path) -> String {
/// fixtures-batch-v1, `dynamo_v2-` under fixtures-stream-v2), ASCENDING by
/// numeric version. Multiple dirs per impl are capture HISTORY (never deleted);
/// readers fold them ascending so the latest capture wins per case.
pub type VersionCaptureSortKey = (Vec<u64>, bool, String);

pub fn version_dirs_ascending(root: &Path, prefix: &str) -> Vec<PathBuf> {
let mut dirs: Vec<(Vec<u64>, PathBuf)> = std::fs::read_dir(root)
let mut dirs: Vec<(VersionCaptureSortKey, PathBuf)> = std::fs::read_dir(root)
.into_iter()
.flatten()
.flatten()
.map(|e| e.path())
.filter(|p| {
p.is_dir()
&& p.file_name().and_then(|n| n.to_str()).is_some_and(|n| {
// `<ver>.patchN` dirs are DISPLAY-ONLY overlays: an OLD parser
// binary re-run to backfill a newer case onto version `<ver>`
// (e.g. dynamo_v2-0.1.11.patch1 = the 0.1.11 binary on streamv2.5.h,
// rendered under the 0.1.11 column in HTML). They are NOT the current
// parser, so they must never join this "latest capture wins" fold —
// otherwise a stale old-binary result can shadow the real latest.
//
// `<ver>+<tag>` dirs are the same kind of thing for the same reason:
// a change-scoped capture, an older build run over the current corpus
// (see `capture_cross_version.rs`). Excluded HERE rather than at each
// call site because the version key below splits on non-digits, so
// `0.1.24+pre163` folds to [0,1,24,163] and would sort ABOVE the real
// 0.1.24 — letting a historical snapshot win "latest capture".
n.starts_with(prefix) && !n.contains(".patch") && !n.contains('+')
})
})
.map(|p| {
let key: Vec<u64> = p
.filter(|p| p.is_dir())
.filter_map(|p| {
let key = p
.file_name()
.and_then(|n| n.to_str())
.and_then(|n| n.strip_prefix(prefix))
.unwrap_or("")
.split(|c: char| !c.is_ascii_digit())
.filter(|s| !s.is_empty())
.map(|s| s.parse().unwrap_or(0))
.collect();
(key, p)
.and_then(|n| version_capture_sort_key(n, prefix))?;
Some((key, p))
})
.collect();
dirs.sort();
dirs.into_iter().map(|(_, p)| p).collect()
}

/// Sort a PR-qualified capture after its matching release. The PR capture represents
/// the current branch, while a plain release remains a historical comparison point.
pub fn version_capture_sort_key(name: &str, prefix: &str) -> Option<VersionCaptureSortKey> {
let version = name.strip_prefix(prefix)?;
// `.patchN` dirs are display-only overlays from an old binary. They fill missing
// cases in that binary's column and must never participate in latest-capture folds.
if version.contains(".patch") {
return None;
}
let base = version.split_once('+').map_or(version, |(base, _)| base);
let numeric = base
.split(|c: char| !c.is_ascii_digit())
.filter(|s| !s.is_empty())
.map(|s| s.parse().unwrap_or(0))
.collect();
Some((numeric, version.contains('+'), version.to_string()))
}

/// One row of the `unified:` block in `conformance/utils/src/parser_families.yaml`.
///
/// That block is the ONE place a family is declared for the unified tab. It replaced
Expand Down
66 changes: 43 additions & 23 deletions conformance/tests/unified_render.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,13 @@
//! Columns: GOLDEN (authored oracle) | vLLM (captured) | Dynamo v2.
//!
//! The Dynamo column is a PER-FAMILY MIXTURE, the same shape as vLLM Rust's: the
//! native UnifiedParser where one exists (qwen3 today), and the v1-reasoning +
//! v2-tool split everywhere else. The split is the "before" state — it parses ALL
//! reasoning first, so reasoning interleaved with tool calls loses its position.
//! native UnifiedParser where one exists, and the v1-reasoning + v2-tool split
//! everywhere else. The split is the "before" state — it parses ALL reasoning
//! first, so reasoning interleaved with tool calls loses its position. Every
//! family in the current golden corpus (gemma4, kimi_k2, muse_glimmer, qwen3)
//! has a native UnifiedParser as of gemma4's migration, so nothing in this
//! corpus exercises the split path today; a new split-only family reintroduces
//! it.
//!
//! Output: `conformance/CONFORMANCE_unified.html` (standalone preview) and
//! `conformance/unified/unified_results.yaml`. The exploder and packager turn that
Expand Down Expand Up @@ -178,12 +182,9 @@ fn unified_delta_json(d: &dynamo_parsers_v2::UnifiedParserEvent) -> Value {

/// Compute the Dynamo v2 unified event list for one input.
///
/// PER-FAMILY MIXTURE, the same shape as vLLM Rust's column: a family that has a
/// unified parser is parsed by it — one state machine per stream owning
/// reasoning + content + tool calls. Every other family still goes through the
/// SPLIT Dynamo serves today: the v1 reasoning parser strips reasoning over the
/// whole stream into ONE assembled field (the merge), then the v2 tool parser
/// streams the leftover content (char-by-char) preserving text/call order.
/// A family with a native unified parser is parsed by one state machine per
/// stream owning reasoning + content + tool calls. All four families in the
/// current corpus take this path; the split fallback remains for other inputs.
///
/// Both paths are driven from the SAME chunking as `dynamo_chunks`, so the
/// assembled row and the per-chunk rows in the popup describe one run.
Expand Down Expand Up @@ -447,7 +448,7 @@ fn classify(family: &str, golden: &[Ev], got: &[Ev]) -> &'static str {
})
.collect()
};
if cat(golden, true) == cat(got, true) && cat(golden, false) == cat(got, false) {
if gc == tc && cat(golden, true) == cat(got, true) && cat(golden, false) == cat(got, false) {
return "ORDER";
}
"LOSS"
Expand Down Expand Up @@ -630,7 +631,7 @@ fn render_unified_conformance_html() {
format!("<hr><div class=todo>{}</div>", esc(todo_for(dclass)))
};
let dcell = cell(
"Dynamo today (v1 reasoning + v2 tool, LIVE)",
"Dynamo today (native unified, LIVE)",
&case.input,
&case.golden,
&events_html(&got),
Expand Down Expand Up @@ -741,9 +742,9 @@ td.c:hover .tip,td.gold:hover .tip{{display:block}}
.legend span{{display:inline-block;padding:2px 8px;border-radius:4px;margin-right:8px;font-size:12px}}
</style>
<h1>Unified conformance — reasoning + content + tool calls, one ordered stream</h1>
<p class=lede>Truth column is <b>GOLDEN</b> — the authored, spec-derived oracle (best-effort error recovery), <i>not</i> captured from any implementation. Both engines are measured against it. <b>Dynamo today</b> is computed LIVE this run (v1 reasoning + v2 tool, composed as Dynamo serves today; Dynamo v2 is moving to a per-family mixture). <b>vLLM 0.25.x</b> is the documented expectation (live capture = U1; only gemma4 uses a native unified parser, the rest are combined). Hover any cell.</p>
<p class=lede>Truth column is <b>GOLDEN</b> — the authored, spec-derived oracle (best-effort error recovery), <i>not</i> captured from any implementation. Both engines are measured against it. <b>Dynamo today</b> is computed LIVE this run with native unified parsers for all four current corpus families. <b>vLLM 0.25.x</b> is the documented expectation (live capture = U1; only gemma4 uses a native unified parser, the rest are combined). Hover any cell.</p>
<p class=legend><span class=MATCH>matches golden</span><span class=RED>diverges (class shown)</span> &nbsp; Dynamo red: <b>{dynamo_red}</b>/{total} · vLLM red: <b>{vllm_red}</b>/{total}</p>
<table><tr><td>case</td><td>GOLDEN</td><td>vLLM 0.25.x<br><span class=sub>(expected)</span></td><td>Dynamo today<br><span class=sub>(v1 reasoning + v2 tool, LIVE)</span></td></tr>
<table><tr><td>case</td><td>GOLDEN</td><td>vLLM 0.25.x<br><span class=sub>(expected)</span></td><td>Dynamo today<br><span class=sub>(native unified, LIVE)</span></td></tr>
{rows}
</table>
<p class=sub>Generated by conformance/tests/unified_render.rs. Cases: conformance/unified/golden_spec/ (authored by gen_unified_golden.py). Taxonomy: conformance/utils/lib/parsers/UNIFIED_CASES.md.</p>
Expand All @@ -763,7 +764,7 @@ td.c:hover .tip,td.gold:hover .tip{{display:block}}
std::fs::create_dir_all(yaml_out.parent().unwrap()).unwrap();
let feed = serde_json::json!({
"schema": "unified-results/v1",
"note": "GOLDEN = authored oracle. dynamo = LIVE (v1 reasoning + v2 tool). vllm = documented expectation (live capture pending U1).",
"note": "GOLDEN = authored oracle. dynamo = LIVE (native unified for all four current corpus families). vllm = documented expectation (live capture pending U1).",
"cases": json_cases,
});
std::fs::write(&yaml_out, serde_yaml::to_string(&feed).unwrap()).unwrap();
Expand All @@ -772,11 +773,19 @@ td.c:hover .tip,td.gold:hover .tip{{display:block}}
out.display()
);

// Sanity: the harness computes REAL failures, not a strawman.
// Sanity: the harness computes REAL failures, not a strawman. vLLM's
// documented-expectation column supplies that signal (`vllm_red` is large
// and asserted elsewhere via the rendered legend), so this only needs to
// pin Dynamo's own invariant.
assert!(total >= 14, "expected the seed corpus");
assert!(
dynamo_red >= 3,
"expected the split to fail the interleaving cases (got {dynamo_red})"
// Every family in the current golden corpus has a native UnifiedParser
// (see the module doc), so Dynamo must match GOLDEN on every case — the
// split's interleaving-order loss no longer applies to anything here. A
// regression back to >0 means either a native parser broke, or a new
// split-only family entered the corpus without a native parser of its own.
assert_eq!(
dynamo_red, 0,
"expected every case to match golden now that every family is native (got {dynamo_red} red)"
);
}

Expand Down Expand Up @@ -841,11 +850,11 @@ fn committed_dynamo_capture_matches_the_live_parsers() {
);
}
// THIS build's capture: the newest version dir, via the shared helper so the
// "which capture is current" rule lives in ONE place. The helper already drops
// `.patchN` overlays and `+tag` change-scoped captures, both of which are older
// parsers and must never be mistaken for the live one. Resolving by readdir order
// instead made this guard nondeterministic — the same commit could pass against one
// shard and report parser drift against another purely on directory listing order.
// "which capture is current" rule lives in ONE place. The helper drops `.patchN`
// overlays, orders a `+tag` immediately before its matching plain release, and still
// lets a newer PR-qualified capture outrank every older release. Resolving by
// readdir order instead made this guard nondeterministic — the same commit could
// pass against one shard and report parser drift against another.
let capture_dir = common::version_dirs_ascending(&root, "dynamo_v2-")
.pop()
.expect("no committed dynamo_v2-<ver> capture dir");
Expand Down Expand Up @@ -920,6 +929,17 @@ fn committed_dynamo_capture_matches_the_live_parsers() {
);
}

#[test]
fn release_qualified_capture_order_preserves_current_and_released_owners() {
let prefix = "dynamo_v2-";
let older_release = common::version_capture_sort_key("dynamo_v2-0.3.2", prefix).unwrap();
let current_pr = common::version_capture_sort_key("dynamo_v2-0.3.3+pr166", prefix).unwrap();
let current_release = common::version_capture_sort_key("dynamo_v2-0.3.3", prefix).unwrap();
assert!(older_release < current_release);
assert!(current_release < current_pr);
assert!(common::version_capture_sort_key("dynamo_v2-0.3.3.patch1", prefix).is_none());
}

fn glob_yaml(dir: &std::path::Path) -> Vec<PathBuf> {
let mut out = Vec::new();
for fam in std::fs::read_dir(dir).into_iter().flatten().flatten() {
Expand Down
2 changes: 1 addition & 1 deletion conformance/utils/src/parser_families.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ markers:
unified:
gemma4:
registry: gemma4
native: false
native: true
reasoning_parser: gemma4
tool_parser: gemma4
golden_spec: gemma4.yaml
Expand Down
82 changes: 82 additions & 0 deletions conformance/utils/tests/test_unified_fixture_overlays.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from pathlib import Path
import sys

import yaml

UTILS_SRC = Path(__file__).resolve().parents[1] / "src"
if str(UTILS_SRC) not in sys.path:
sys.path.insert(0, str(UTILS_SRC))

import generate_conformance_table as table # noqa: E402


def _write_case(root, dirname, family, key, body, **metadata):
path = root / dirname / family / f"{key}.yaml"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
yaml.safe_dump(
{"family": family, "mode": "unified", **metadata, "cases": {key: body}},
sort_keys=False,
)
)


def test_sparse_unified_patch_merges_with_base_and_overrides_in_order(tmp_path):
family = "gemma4"
base_key = "UNIFIED.1.a"
patch_key = "UNIFIED.1.b"
for key, scenario in (
(base_key, "base_case"),
(patch_key, "patch_case"),
):
_write_case(
tmp_path,
"inputs",
family,
key,
{"scenario": scenario, "chunks": [{"delta_text": scenario}]},
model_label=family,
)
_write_case(
tmp_path,
"golden",
family,
key,
{"assembled": [{"kind": "text", "text": scenario}]},
captured_with={"golden": "v1"},
)

_write_case(
tmp_path,
"dynamo_v2-0.2.1",
family,
base_key,
{"assembled": [{"kind": "text", "text": "base"}], "chunks": []},
captured_with={"dynamo_v2": "0.2.1"},
)
_write_case(
tmp_path,
"dynamo_v2-0.2.1.patch1",
family,
base_key,
{"assembled": [{"kind": "text", "text": "patched"}], "chunks": []},
captured_with={"dynamo_v2": "0.2.1.patch1"},
)
_write_case(
tmp_path,
"dynamo_v2-0.2.1.patch1",
family,
patch_key,
{"assembled": [{"kind": "text", "text": "patch-only"}], "chunks": []},
captured_with={"dynamo_v2": "0.2.1.patch1"},
)

cases, _caps, versions = table._load_unified_fixtures(tmp_path)
by_scenario = {case["scenario"]: case for case in cases}

assert versions["dynamo_v2_all"] == ["0.2.1"]
assert by_scenario["base_case"]["dynamo_by_ver"]["0.2.1"]["assembled"][0]["text"] == "patched"
assert by_scenario["patch_case"]["dynamo_by_ver"]["0.2.1"]["assembled"][0]["text"] == "patch-only"
2 changes: 1 addition & 1 deletion parsers/v2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ Reasoning families:
| **Gemma 4 channel** | `<\|channel>thought\n...<channel\|>` with role-label prefix stripped | `gemma4_parser.rs` | Google Gemma 4 thinking models |
| **ATEM channel** | `<\|start\|>assistant to=self<\|message\|>...<\|eom\|>` — the recipient picks the channel, so there is no delimiter pair | `muse_glimmer_parser.rs` | Muse-Glimmer-30B |

Streaming (v2) implementations exist today for `harmony`, `deepseek_v4` (DSML), `qwen3_coder`, and `muse_glimmer` (ATEM); the remaining families run on the v1 batch parser until their streaming port lands.
Streaming (v2) implementations exist today for `harmony`, `deepseek_v4` (DSML), `qwen3_coder`, `gemma4`, and `muse_glimmer` (ATEM); the remaining families run on the v1 batch parser until their streaming port lands.

## Why It Mimics vLLM Rust

Expand Down
Loading
Loading