[Bugfix] Bound the grammar-compile thread pool and expose SGLANG_GRAMMAR_COMPILE_MAX_WORKERS

python/sglang/srt/constrained/base_grammar_backend.py

@@ -15,13 +15,15 @@
 
 import json
 import logging
+import os
 import time
 from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass, field
 from typing import Dict, List, NamedTuple, Optional, Tuple
 
 import torch
 
+from sglang.srt.environ import envs
 from sglang.srt.parser.reasoning_parser import ReasoningParser
 from sglang.srt.runtime_context import (
     get_context,
@@ -56,7 +58,6 @@ class GrammarRow(NamedTuple):
 
 
 class BaseGrammarObject:
-
     def __init__(self):
         self._finished = False
         self.grammar_stats = None
@@ -199,11 +200,36 @@ def __repr__(self):
         return f"InvalidGrammarObject(error_message={self.error_message!r})"
 
 
+def get_grammar_compile_max_workers(cpu_count: Optional[int] = None) -> int:
+    """Number of workers for the grammar-compilation thread pool.
+
+    Grammar compilation is CPU-bound and runs inside the scheduler process,
+    so the pool is capped at 8 workers instead of using the
+    ThreadPoolExecutor default of min(32, cpu_count + 4): os.cpu_count() is
+    not cgroup-aware, so inside a container with a CFS quota (e.g. a
+    Kubernetes pod with a low cpu limit scheduled on a many-core host) a
+    large pool oversubscribes the quota, triggers CFS throttling, and stalls
+    the scheduler's decode loop. Each compile can additionally spawn the
+    grammar backend's internal threads (xgrammar defaults to max_threads=8
+    per compile), multiplying the oversubscription.
+
+    Set SGLANG_GRAMMAR_COMPILE_MAX_WORKERS to a positive value to override.
+    """
+    override = envs.SGLANG_GRAMMAR_COMPILE_MAX_WORKERS.get()
+    if override > 0:
+        return override
+    if cpu_count is None:
+        cpu_count = os.cpu_count() or 1
+    return max(1, min(cpu_count // 2, 8))
+
+
 class BaseGrammarBackend:
     _enable_strict_thinking: bool = False
 
     def __init__(self):
-        self.executor = ThreadPoolExecutor()
+        self.executor = ThreadPoolExecutor(
+            max_workers=get_grammar_compile_max_workers()
+        )
         self.cache: Dict[Tuple[str, str], BaseGrammarObject] = {}
 
     def initialize_vocab_mask_buffer(

python/sglang/srt/environ.py

@@ -376,6 +376,12 @@ class Envs:
     # ===================================================================
     SGLANG_GRAMMAR_POLL_INTERVAL = EnvFloat(0.005)
     SGLANG_GRAMMAR_MAX_POLL_ITERATIONS = EnvInt(10000)
+    # Max workers for the grammar-compilation thread pool. 0 (default) sizes
+    # the pool as min(cpu_count // 2, 8). The pool is deliberately capped:
+    # os.cpu_count() is not cgroup-aware, so sizing by host CPU count
+    # oversubscribes container CPU quotas (CFS throttling) and stalls the
+    # scheduler while grammars compile.
+    SGLANG_GRAMMAR_COMPILE_MAX_WORKERS = EnvInt(0)
     SGLANG_DISABLE_OUTLINES_DISK_CACHE = EnvBool(False)
 
     # ===================================================================

test/registered/unit/constrained/test_base_grammar_backend.py

@@ -16,6 +16,7 @@
 """
 
 import json
+import os
 import unittest
 from concurrent.futures import Future
 from unittest.mock import MagicMock, patch
@@ -27,6 +28,7 @@
     GrammarStats,
     InvalidGrammarObject,
     create_grammar_backend,
+    get_grammar_compile_max_workers,
     register_grammar_backend,
 )
 from sglang.srt.runtime_context import get_context  # noqa: E402
@@ -524,5 +526,44 @@ def test_nul_free_payload_still_dispatches(self):
                 dispatch.assert_called_once_with(key_string)
 
 
+class TestGrammarCompileMaxWorkers(unittest.TestCase):
+    """Test grammar-compile thread pool sizing (container/cgroup safety cap).
+
+    os.cpu_count() is not cgroup-aware, so the pool must not scale with the
+    host CPU count: on a many-core host inside a low-cpu-limit container an
+    oversized pool oversubscribes the CFS quota and stalls the scheduler.
+    """
+
+    def test_capped_at_8(self):
+        for cpu_count, expected in [
+            (1, 1),
+            (2, 1),
+            (4, 2),
+            (16, 8),
+            (17, 8),
+            (172, 8),  # many-core host: must stay capped
+        ]:
+            self.assertEqual(get_grammar_compile_max_workers(cpu_count), expected)
+
+    def test_defaults_to_host_cpu_count(self):
+        with patch("os.cpu_count", return_value=32):
+            self.assertEqual(get_grammar_compile_max_workers(), 8)
+
+    def test_env_override(self):
+        with patch.dict(os.environ, {"SGLANG_GRAMMAR_COMPILE_MAX_WORKERS": "4"}):
+            self.assertEqual(get_grammar_compile_max_workers(172), 4)
+
+    def test_env_zero_means_auto(self):
+        with patch.dict(os.environ, {"SGLANG_GRAMMAR_COMPILE_MAX_WORKERS": "0"}):
+            self.assertEqual(get_grammar_compile_max_workers(172), 8)
+
+    def test_at_least_one_worker(self):
+        self.assertEqual(get_grammar_compile_max_workers(0), 1)
+
+    def test_backend_executor_is_bounded(self):
+        backend = BaseGrammarBackend()
+        self.assertLessEqual(backend.executor._max_workers, 8)
+
+
 if __name__ == "__main__":
     unittest.main()