[2.0] Reject unknown benchmark.type at load; delete dead BenchmarkType enum

src/srtctl/core/schema.py

@@ -310,19 +310,6 @@ class Precision(str, Enum):
     BF16 = "bf16"
 
 
-class BenchmarkType(str, Enum):
-    MANUAL = "manual"
-    CUSTOM = "custom"
-    SA_BENCH = "sa-bench"
-    ROUTER = "router"
-    MOONCAKE_ROUTER = "mooncake-router"
-    TRACE_REPLAY = "trace-replay"
-    MMLU = "mmlu"
-    GPQA = "gpqa"
-    GSM8K = "gsm8k"
-    LONGBENCHV2 = "longbenchv2"
-
-
 class ProfilingType(str, Enum):
     NSYS = "nsys"
     TORCH = "torch"
@@ -1905,8 +1892,29 @@ def __post_init__(self):
         self._validate_static_router_frontend()
         self._validate_dynamo_sidecar()
         self._validate_host_setup()
+        self._validate_benchmark_type()
         self._warn_dp_launch_mode()
 
+    def _validate_benchmark_type(self) -> None:
+        """Reject a benchmark.type that no runner is registered for.
+
+        An unknown type (a typo like ``gsm8k-bench``, or a removed one) currently
+        loads fine and only fails deep in the benchmark stage after a full
+        allocation. Catch it at load time against the registry, plus the special
+        ``manual`` type (no runner; the server just comes up ready). Import is
+        lazy and guarded so a registry import hiccup never blocks a load.
+        """
+        btype = self.benchmark.type
+        try:
+            import srtctl.benchmarks  # noqa: F401 - importing the package registers every runner
+            from srtctl.benchmarks.base import list_benchmarks
+
+            allowed = set(list_benchmarks()) | {"manual"}
+        except Exception:  # noqa: BLE001 - never block a config load on the registry import
+            return
+        if btype not in allowed:
+            raise ValueError(f"Unknown benchmark.type {btype!r}. Available: {', '.join(sorted(allowed))}")
+
     def _validate_host_setup(self) -> None:
         """Reject host_setup blocks that would fail or hang mid-job.
 

tests/test_configs.py

@@ -5073,3 +5073,38 @@ def test_recipe_without_gpu_type_and_no_cluster_default_loads(self):
         config = SrtConfig.Schema().load(self._recipe({"agg_nodes": 1, "agg_workers": 1}))
         assert config.resources.gpu_type is None
         assert config.resources.gpus_per_node == 4
+
+
+class TestBenchmarkTypeValidation:
+    """benchmark.type must name a registered runner (or 'manual')."""
+
+    def _recipe(self, benchmark: dict) -> dict:
+        return {
+            "name": "bench-type",
+            "model": {"path": "/m", "container": "/c.sqsh", "precision": "fp8"},
+            "resources": {"gpu_type": "h100", "gpus_per_node": 8, "agg_nodes": 1, "agg_workers": 1},
+            "benchmark": benchmark,
+        }
+
+    def test_registered_and_manual_types_load(self):
+        from srtctl.core.schema import SrtConfig
+
+        for btype in ("manual", "sa-bench", "custom", "mmlu", "trace-replay", "mooncake-router"):
+            benchmark = {"type": btype}
+            if btype == "custom":
+                benchmark["command"] = "echo hi"
+            config = SrtConfig.Schema().load(self._recipe(benchmark))
+            assert config.benchmark.type == btype
+
+    def test_unknown_type_is_rejected_at_load(self):
+        import pytest
+
+        from srtctl.core.schema import SrtConfig
+
+        with pytest.raises(Exception, match="gsm8k-bench"):
+            SrtConfig.Schema().load(self._recipe({"type": "gsm8k-bench"}))
+
+    def test_benchmark_type_enum_is_gone(self):
+        import srtctl.core.schema as schema_mod
+
+        assert not hasattr(schema_mod, "BenchmarkType")