fix: reject grammar atomic tokens via byte path when atomic match fails

[arpera]

There was found a bug when xgrammar accepts invalid json object {"queries":["hello","]}. This json missed " close bracket for the second element in the array. The bug in xgrammar was in AcceptToken that did byte-by-byte processing for atomic tokens. This PR fixes this issue introducing a special set of all atomic tokens and forbids to process them byte-by-byte.

Testing Done

import xgrammar, json
from transformers import AutoTokenizer

tok = AutoTokenizer.from_pretrained("moonshotai/Kimi-K2.5", trust_remote_code=True)
tok_info = xgrammar.TokenizerInfo.from_huggingface(tok, vocab_size=163840)
compiler = xgrammar.GrammarCompiler(tok_info)

def old_api():
    schema = '{"properties":{"queries":{"items":{"type":"string"},"type":"array"}},"required":["queries"],"type":"object"}'
    tag = xgrammar.StructuralTagItem(
        begin="<|tool_calls_section_begin|><|tool_call_begin|>functions.search:0<|tool_call_argument_begin|>",
        schema=schema,
        end="<|tool_call_end|><|tool_calls_section_end|>",
    )
    grammar = compiler.compile_structural_tag([tag], ["<|tool_calls_section_begin|>"])
    return grammar

def new_api():
    schema = {
        "type": "object",
        "properties": {
            "queries": {
                "type": "array",
                "items": {"type": "string"},
            },
        },
        "required": ["queries"],
    }
    structural_tag = {
        "type": "structural_tag",
        "format": {
            "type": "sequence",
            "elements": [
                {"type": "token", "token": "<|tool_calls_section_begin|>"},
                {"type": "token", "token": "<|tool_call_begin|>"},
                {"type": "const_string", "value": "functions.search:0"},
                {"type": "token", "token": "<|tool_call_argument_begin|>"},
                {"type": "json_schema", "json_schema": schema},
                {"type": "token", "token": "<|tool_call_end|>"},
                {"type": "token", "token": "<|tool_calls_section_end|>"},
            ],
        },
    }
    grammar = compiler.compile_structural_tag(structural_tag)
    return grammar

# This INVALID JSON is incorrectly accepted:
text = '<|tool_calls_section_begin|><|tool_call_begin|>functions.search:0<|tool_call_argument_begin|>{"queries":["hello","]}' + \
       '<|tool_call_end|><|tool_calls_section_end|>'
ids = tok.encode(text, allowed_special="all")
grammar = new_api()
matcher = xgrammar.GrammarMatcher(grammar)
all_accepted = all(matcher.accept_token(tid) for tid in ids)
if not all_accepted:
    print("OK")
else:
    print("FAIL")
    print(f"JSON valid: {json.loads('{\"queries\":[\"hello\",\"]}')}")  # raises JSONDecodeError

This test fails on current xgrammar but passes with this PR.

N.B. The bug still persists in case of old API. If old API is deprecated now then consider highlighting that for users since now it is also possible to use old API and the bug persists for it.