fix(parsers-v2): reserve V4.1 raw-string closing prefixes

parsers/v2/src/structural_tag/builders/deepseek_dsml.rs

@@ -92,6 +92,7 @@ impl DeepSeekDsmlFormat {
                     style: self.argument_style(),
                     any_order: tool_arguments_any_order,
                     max_whitespace_cnt: None,
+                    excludes: Vec::new(),
                 })),
                 end: [self.invoke_end(), "\n"].concat(),
             })

parsers/v2/src/structural_tag/builders/harmony.rs

@@ -23,6 +23,7 @@ impl HarmonyGrammar {
                 style: JsonSchemaStyle::Json,
                 any_order: false,
                 max_whitespace_cnt: None,
+                excludes: Vec::new(),
             })),
             // The engine consumes the turn stop after the JSON grammar completes.
             end: String::new(),

parsers/v2/src/structural_tag/builders/minimax_m3.rs

@@ -78,6 +78,7 @@ impl MiniMaxM3 {
                             // between fields indefinitely. String contents remain
                             // unconstrained by this formatting limit.
                             max_whitespace_cnt: Some(1),
+                            excludes: Vec::new(),
                         })),
                         end: INVOKE_END.into(),
                     })

parsers/v2/src/structural_tag/builders/mod.rs

@@ -113,7 +113,7 @@ impl StructuralTagBuilder {
             .exclude_special_tokens
             .unwrap_or_else(|| grammar.default_exclude_special_tokens());
 
-        let format = match policy {
+        let mut format = match policy {
             ResolvedToolCalling::Disabled => {
                 if let Some(schema) = context.structured_output_schema {
                     grammar.build_structured_output(schema)
@@ -155,6 +155,9 @@ impl StructuralTagBuilder {
                 }
             }
         };
+        if exclude_special_tokens {
+            super::wire::reserve_v41_closing_prefix(&mut format);
+        }
         let format = if options.reasoning_boundary == ReasoningBoundary::StructuralTag {
             wrap_reasoning_if_needed(
                 context.starts_in_reasoning,
@@ -205,6 +208,7 @@ pub(super) trait ToolCallGrammar: Send + Sync {
             style: JsonSchemaStyle::Json,
             any_order: false,
             max_whitespace_cnt: None,
+            excludes: Vec::new(),
         })
     }
 

parsers/v2/src/structural_tag/builders/muse_spark.rs

@@ -207,6 +207,7 @@ fn value_format(schema: &Value, root: &Value, any_order: bool) -> anyhow::Result
         style: JsonSchemaStyle::Json,
         any_order,
         max_whitespace_cnt: None,
+        excludes: Vec::new(),
     }))
 }
 
@@ -409,6 +410,7 @@ impl MuseSparkGrammar {
             style: JsonSchemaStyle::Json,
             any_order: false,
             max_whitespace_cnt: None,
+            excludes: Vec::new(),
         }))
     }
 

parsers/v2/src/structural_tag/builders/templated_tool_calls.rs

@@ -163,6 +163,7 @@ impl TemplatedToolCallFormat {
                 style: self.arguments_style,
                 any_order: tool_arguments_any_order,
                 max_whitespace_cnt: None,
+                excludes: Vec::new(),
             })
         };
 

parsers/v2/src/structural_tag/context.rs

@@ -42,6 +42,9 @@ pub enum ReasoningBoundary {
 #[serde(default, deny_unknown_fields)]
 pub struct StructuralTagOptions {
     /// Overrides whether reasoning and tool-call markers are excluded from free-text regions.
+    /// For DeepSeek V4.1 this also reserves the `</｜` closing-control prefix inside
+    /// raw tool-argument strings. Set `false` when an argument must contain that literal
+    /// prefix. XGrammar drops `minLength`/`maxLength` on strings while exclusions are active.
     pub exclude_special_tokens: Option<bool>,
     /// Defaults to [`ReasoningBoundary::StructuralTag`].
     pub reasoning_boundary: ReasoningBoundary,

parsers/v2/src/structural_tag/mod.rs

@@ -12,6 +12,12 @@
 //! format node. Check the version bundled with the inference backend.
 //! XGrammar 0.1.32 also ignores JSON Schema `any_order`, so it does not honor
 //! reordered JSON keys requested by [`StructuralTagOptions::tool_arguments_any_order`].
+//!
+//! DeepSeek V4.1 tool arguments reserve the `</｜` closing-control prefix with
+//! `JSONSchemaFormat.excludes` when special-token exclusion is enabled. This is a
+//! generation restriction, not parser recovery. XGrammar 0.2.8 honors the exclusion but
+//! drops `minLength`/`maxLength` on strings while it is active. A backend that preserves
+//! those bounds is needed before the exclusion can be relied on with bounded strings.
 
 pub(crate) mod builders;
 mod context;

parsers/v2/src/structural_tag/tests.rs

@@ -975,3 +975,82 @@ fn invalid_required_and_named_requests_fail() {
     };
     assert!(QWEN3_CODER.build(&named).is_err());
 }
+
+fn json_schema_formats(value: &Value) -> Vec<&Value> {
+    fn visit<'a>(value: &'a Value, result: &mut Vec<&'a Value>) {
+        match value {
+            Value::Object(fields) => {
+                if fields.get("type") == Some(&json!("json_schema")) {
+                    result.push(value);
+                }
+                for child in fields.values() {
+                    visit(child, result);
+                }
+            }
+            Value::Array(children) => {
+                for child in children {
+                    visit(child, result);
+                }
+            }
+            _ => {}
+        }
+    }
+    let mut result = Vec::new();
+    visit(value, &mut result);
+    result
+}
+
+#[test]
+fn v41_argument_exclusion_is_scoped_and_honors_the_opt_out() {
+    let tools = tools();
+    let response_schema = json!({"type": "string", "const": "literal </｜ marker"});
+    for (family, builder) in [
+        ("deepseek_v41", &DEEPSEEK_DSML_V41),
+        ("deepseek_v4", &DEEPSEEK_DSML),
+        ("qwen3_coder", &QWEN3_CODER),
+        ("glm47", &GLM47),
+    ] {
+        for choice in [
+            StructuralTagToolChoice::Auto,
+            StructuralTagToolChoice::Required,
+            StructuralTagToolChoice::Named("get_weather"),
+        ] {
+            for exclude_special_tokens in [None, Some(true), Some(false)] {
+                for starts_in_reasoning in [false, true] {
+                    let tag = builder
+                        .build_with_options(
+                            &StructuralTagContext {
+                                tools: &tools,
+                                tool_choice: choice,
+                                parallel_tool_calls: Some(false),
+                                schema_mode: StructuralTagSchemaMode::Strict,
+                                structured_output_schema: Some(&response_schema),
+                                starts_in_reasoning,
+                            },
+                            &StructuralTagOptions {
+                                exclude_special_tokens,
+                                ..Default::default()
+                            },
+                        )
+                        .unwrap()
+                        .unwrap();
+                    let schemas = json_schema_formats(&tag);
+                    assert!(!schemas.is_empty(), "{family}: {tag}");
+                    for schema in schemas {
+                        let guarded = family == "deepseek_v41"
+                            && exclude_special_tokens != Some(false)
+                            && schema["style"] == "deepseek_v4_1_xml";
+                        if guarded {
+                            assert_eq!(schema["excludes"], json!(["</｜"]));
+                        } else {
+                            assert!(schema.get("excludes").is_none(), "{family}: {schema}");
+                        }
+                        if schema["style"] == "json" {
+                            assert_eq!(schema["json_schema"], response_schema);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

parsers/v2/src/structural_tag/wire.rs

@@ -109,6 +109,9 @@ pub(crate) struct JsonSchemaFormat {
     pub any_order: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub max_whitespace_cnt: Option<usize>,
+    /// Substrings that must not occur inside string values.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub excludes: Vec<String>,
 }
 
 fn is_false(value: &bool) -> bool {
@@ -188,6 +191,7 @@ pub(super) fn import_native(
             },
             any_order,
             max_whitespace_cnt: None,
+            excludes: Vec::new(),
         }),
         native::Format::AnyTokens(f) => Format::AnyTokens(f),
         native::Format::AnyText(f) => Format::AnyText(AnyTextFormat {
@@ -221,3 +225,41 @@ pub(super) fn import_tag(
         end: tag.end,
     }
 }
+
+/// Reserve V4.1 closing-control prefixes in tool-argument strings. A final-response
+/// JSON schema has `Json` style and must not inherit this restriction.
+pub(super) fn reserve_v41_closing_prefix(format: &mut Format) {
+    match format {
+        Format::JsonSchema(schema) => {
+            if matches!(schema.style, JsonSchemaStyle::DeepseekV41Xml) {
+                schema.excludes.push("</｜".to_string());
+            }
+        }
+        Format::Tag(tag) => reserve_v41_closing_prefix(&mut tag.content),
+        Format::TriggeredTags(tags) => {
+            for tag in &mut tags.tags {
+                reserve_v41_closing_prefix(&mut tag.content);
+            }
+        }
+        Format::TagsWithSeparator(tags) => {
+            for tag in &mut tags.tags {
+                reserve_v41_closing_prefix(&mut tag.content);
+            }
+        }
+        Format::Sequence(sequence) => {
+            for element in &mut sequence.elements {
+                reserve_v41_closing_prefix(element);
+            }
+        }
+        Format::Or(alternatives) => {
+            for element in &mut alternatives.elements {
+                reserve_v41_closing_prefix(element);
+            }
+        }
+        Format::Optional(nested) | Format::Star(nested) => {
+            reserve_v41_closing_prefix(&mut nested.content);
+        }
+        Format::Repeat(repeat) => reserve_v41_closing_prefix(&mut repeat.content),
+        Format::ConstString(_) | Format::Regex(_) | Format::AnyText(_) | Format::AnyTokens(_) => {}
+    }
+}

parsers/v2/tests/check_structural_tag_xgrammar.py

@@ -18,6 +18,9 @@
 schema validation and native recovery are separate contracts. Harmony checks
 qualify grammar strings separately because its terminal <|call|> is engine-owned.
 The local byte vocabulary qualifies strings, not production tokenizer masks.
+A request may instead provide ``tokens``, decoded token strings appended to the
+byte vocabulary and fed through ``accept_token``. Every result reports the
+backend version and whether it honors ``JSONSchemaFormat.excludes``.
 """
 
 import importlib.util
@@ -48,32 +51,93 @@
 SPEC.loader.exec_module(NATIVE)
 
 
+BYTE_VOCAB = [bytes([i]) for i in range(256)]
+
+
+def tokenizer_for(token_texts=()):
+    return NATIVE.TokenizerInfo(
+        BYTE_VOCAB + [text.encode("utf-8") for text in token_texts], 0, None, [], False
+    )
+
+
+def compiler_for(tokenizer):
+    return NATIVE.GrammarCompiler(tokenizer, 1, False, -1)
+
+
+def excludes_supported():
+    probe = {
+        "type": "structural_tag",
+        "format": {
+            "type": "tag",
+            "begin": "",
+            "content": {
+                "type": "json_schema",
+                "json_schema": {
+                    "type": "object",
+                    "properties": {"text": {"type": "string"}},
+                    "required": ["text"],
+                    "additionalProperties": False,
+                },
+                "style": "deepseek_v4_1_xml",
+                "excludes": ["probe"],
+            },
+            "end": "",
+        },
+    }
+    text = (
+        "<｜DSML｜ parameter name=\"text\" string=\"true\">"
+        "probe</｜DSML｜ parameter>"
+    )
+    try:
+        grammar = compiler_for(tokenizer_for()).compile_structural_tag(json.dumps(probe))
+        matcher = NATIVE.GrammarMatcher(grammar, None, True, 0, None)
+        return not matcher.accept_string(text, False)
+    except Exception:
+        return False
+
+
 def main():
     requests = json.load(sys.stdin)
     if not requests:
         raise ValueError("no grammar cases received")
-    tokenizer = NATIVE.TokenizerInfo([bytes([i]) for i in range(256)], 0, None, [], False)
-    compiler = NATIVE.GrammarCompiler(tokenizer, 1, False, -1)
+    byte_compiler = compiler_for(tokenizer_for())
+    supports_excludes = excludes_supported()
     results = []
     for request in requests:
         try:
+            if "tokens" in request:
+                tokens = request["tokens"]
+                compiler = compiler_for(tokenizer_for(tokens))
+            else:
+                compiler = byte_compiler
             grammar = compiler.compile_structural_tag(json.dumps(request["grammar"]))
         except Exception as error:
             raise RuntimeError(f"compile {request['id']}: {error}") from error
         matcher = NATIVE.GrammarMatcher(grammar, None, True, 0, None)
-        accepted = matcher.accept_string(request["input"], False)
-        rejected_at = None
-        if not accepted:
-            diagnostic = NATIVE.GrammarMatcher(grammar, None, True, 0, None)
-            for offset, character in enumerate(request["input"]):
-                if not diagnostic.accept_string(character, False):
-                    rejected_at = request["input"][: offset + 1]
+        if "tokens" in request:
+            accepted = True
+            rejected_at = None
+            for index in range(len(tokens)):
+                if not matcher.accept_token(len(BYTE_VOCAB) + index, False):
+                    accepted = False
+                    rejected_at = "".join(tokens[: index + 1])
                     break
+        else:
+            accepted = matcher.accept_string(request["input"], False)
+            rejected_at = None
+            if not accepted:
+                diagnostic = NATIVE.GrammarMatcher(grammar, None, True, 0, None)
+                for offset, character in enumerate(request["input"]):
+                    if not diagnostic.accept_string(character, False):
+                        rejected_at = request["input"][: offset + 1]
+                        break
         results.append({
             "id": request["id"],
             "accepted": accepted,
             "complete": accepted and matcher.is_completed(),
             "rejected_prefix": rejected_at,
+            "backend_version": EXPECTED_VERSION,
+            "excludes_supported": supports_excludes,
         })
     json.dump(results, sys.stdout)