diff --git a/inferencex-e2e/docs/agentx-standalone.md b/inferencex-e2e/docs/agentx-standalone.md index 26284994c2..4f2c7a0873 100644 --- a/inferencex-e2e/docs/agentx-standalone.md +++ b/inferencex-e2e/docs/agentx-standalone.md @@ -43,6 +43,14 @@ environment variables override preset defaults. Model, tokenizer, dataset, concurrency, and output remain caller inputs; `--tokenizer-trust-remote-code` is needed only for tokenizers with custom code. +InferenceX's `python3 -m infx.bench agentic` wrapper defaults to the full, +uncapped corpus (`semianalysis_cc_traces_weka_062126`). Model prefixes beginning +with `dsr1`, `qwen3.5`, or `qwen3.8next` explicitly select the +256k-capped corpus (`semianalysis_cc_traces_weka_062126_256k`). New model families +use the full corpus unless added to `CAPPED_FAMILIES` in +`infx/bench/agentic/traces.py`. `WEKA_LOADER_OVERRIDE` can pin either loader +regardless of model family; it does not configure the standalone `agentx` CLI. + The default random seed is **42**. Keep this seed when reproducing AgentX results, or pass `--random-seed 123` to use a different seed. diff --git a/inferencex-e2e/infx/bench/agentic/traces.py b/inferencex-e2e/infx/bench/agentic/traces.py index 021991e4e3..45a2a885ff 100644 --- a/inferencex-e2e/infx/bench/agentic/traces.py +++ b/inferencex-e2e/infx/bench/agentic/traces.py @@ -10,12 +10,12 @@ UNCAPPED: "semianalysisai/cc-traces-weka-062126", CAPPED: "semianalysisai/cc-traces-weka-062126-256k", } -UNCAPPED_FAMILIES = ("dsv4", "glm5.2", "glm5.3", "minimaxm3", "kimik3") +CAPPED_FAMILIES = ("dsr1", "qwen3.5", "qwen3.8next") def resolve(model_prefix: str, override: str | None) -> tuple[str, str]: """``(loader, hf_dataset)``; ``override`` (``WEKA_LOADER_OVERRIDE``) pins either corpus.""" - loader = override or (UNCAPPED if model_prefix.startswith(UNCAPPED_FAMILIES) else CAPPED) + loader = override or (CAPPED if model_prefix.startswith(CAPPED_FAMILIES) else UNCAPPED) if loader not in LOADERS: raise InputError(f"unknown WEKA_LOADER_OVERRIDE={loader!r}; allowed: {', '.join(LOADERS)}") return loader, LOADERS[loader] diff --git a/inferencex-e2e/infx/tests/bench/test_agentic_command.py b/inferencex-e2e/infx/tests/bench/test_agentic_command.py index 260809124b..a60d67d2fe 100644 --- a/inferencex-e2e/infx/tests/bench/test_agentic_command.py +++ b/inferencex-e2e/infx/tests/bench/test_agentic_command.py @@ -15,7 +15,6 @@ import pytest -from infx.bench.agentic import traces from infx.bench.agentic.run import Plan, execute from infx.bench.agentic.venv import Runtime from infx.bench.env import BenchError, InputError @@ -426,7 +425,7 @@ def frontend(method: str, path: str) -> tuple[int, object]: assert result.returncode == 0, result.stderr assert _events(tmp_path) == [ - f"hf download --repo-type dataset {traces.LOADERS[traces.CAPPED]}", + "hf download --repo-type dataset semianalysisai/cc-traces-weka-062126", "GET /v1/models", "GET /v1/chat/completions", "replay", diff --git a/inferencex-e2e/infx/tests/bench/test_agentic_replay.py b/inferencex-e2e/infx/tests/bench/test_agentic_replay.py index e3f7a1c847..46eb7d2163 100644 --- a/inferencex-e2e/infx/tests/bench/test_agentic_replay.py +++ b/inferencex-e2e/infx/tests/bench/test_agentic_replay.py @@ -40,7 +40,7 @@ def test_default_point_leaves_the_replay_policy_to_the_scenario(): "--model", "deepseek-ai/DeepSeek-V4-Pro", "--tokenizer", "deepseek-ai/DeepSeek-V4-Pro", "--concurrency", "8", "--benchmark-duration", "3600", "--tokenizer-trust-remote-code", "--output-artifact-dir", "/results/aiperf_artifacts", - "--public-dataset", traces.CAPPED, + "--public-dataset", "semianalysis_cc_traces_weka_062126", ] # fmt: skip @@ -168,16 +168,37 @@ def test_malformed_replay_inputs_are_rejected(overrides, message): @pytest.mark.parametrize( - ("prefix", "override", "loader"), + ("prefix", "override", "capped"), [ - ("big41flash", None, traces.UNCAPPED), - ("other", None, traces.CAPPED), - ("other", traces.UNCAPPED, traces.UNCAPPED), + ("dsr1", None, True), + ("qwen3.5", None, True), + ("qwen3.8next", None, True), + ("musespark", None, False), + ("qwen3.5-variant", None, True), + ("dsv4", None, False), + ("dsv41flash", None, False), + ("glm5.2", None, False), + ("glm5.3", None, False), + ("minimaxm3", None, False), + ("kimik3", None, False), + ("new-model", None, False), + ("qwen3.9", None, False), + ("new-model", "", False), + ("qwen3.5", traces.UNCAPPED, False), + ("new-model", traces.CAPPED, True), ], ) -def test_corpus_follows_the_model_family_context_unless_pinned( - monkeypatch, prefix, override, loader -): - monkeypatch.setattr(traces, "UNCAPPED_FAMILIES", ("big4",)) +def test_corpus_is_uncapped_unless_explicitly_capped_or_pinned(prefix, override, capped): + loader, dataset = ( + ("semianalysis_cc_traces_weka_062126_256k", "semianalysisai/cc-traces-weka-062126-256k") + if capped + else ("semianalysis_cc_traces_weka_062126", "semianalysisai/cc-traces-weka-062126") + ) + env = {**BASE_ENV, "MODEL_PREFIX": prefix} + if override is not None: + env["WEKA_LOADER_OVERRIDE"] = override + + config = ReplayConfig.from_env(env, Path("/results")) - assert traces.resolve(prefix, override) == (loader, traces.LOADERS[loader]) + assert (config.loader, config.dataset) == (loader, dataset) + assert _value(replay_argv(config, "/venv/bin/agentx"), "--public-dataset") == loader diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 5737704c28..c16125ad20 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10097,3 +10097,18 @@ - "Move the complete GB300 Kimi-K3 AgentX curve to vLLM nightly-49cb2bbf, use probabilistic DSpark sampling throughout (golden AL 3.36/3.62/3.84 for 4/5/7 drafts), and add low-latency megakernel points." - "Pin Dynamo ffbbaaa6 (dynamo-parsers 8.1.0), use Kimi-K3 parsers only in evaluation recipes, and set DYN_TCP_REQUEST_TIMEOUT=120 for the frontend and workers." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4208 + +- config-keys: + - musespark-fp4-mi355x-sglang-agentic-dflash + - musespark-fp4-gb200-sglang-agentic-dflash + - musespark-fp4-b200-sglang-agentic-dflash + - musespark-fp4-h200-sglang-agentic-dflash + - musespark-fp4-gb300-sglang-agentic-dflash + - musespark-fp4-b300-sglang-agentic-dflash + - musespark-fp4-h100-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Default AgentX trace selection to the full corpus, retaining explicit 256k exceptions for dsr1, qwen3.5, and qwen3.8next." + - "Move Muse Spark Open AgentX points from the 256k-capped corpus to semianalysis_cc_traces_weka_062126; preserve explicit loader overrides." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4287