diff --git a/MODELS.md b/MODELS.md index 060c053b87..21cc16d682 100644 --- a/MODELS.md +++ b/MODELS.md @@ -150,6 +150,7 @@ Other offloading tiers, including NVMe KV cache offloading, are outside the init | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| +| DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | Agentic coding (DSpark, Engram UVA offload; GPU validation pending) | | | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26 ([#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)) | Agentic coding | | | Kimi-K3 | `kimik3` | 2026-07-27 ([#2391](https://github.com/SemiAnalysisAI/InferenceX/pull/2391)) | Agentic coding (DSpark only) | Agentic coding non-DSpark arm (deprecated from day 0) | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding (the non-MTP arm still runs while the MTP-only transition remains pending, as explained in the Deprecation Notice) | | diff --git a/MODELS_zh.md b/MODELS_zh.md index 5b674b4607..2ab04fae2e 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -150,6 +150,7 @@ InferenceX 支持 SGLang 和 vLLM 双方的维护者,并响应 AI 实验室和 | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| +| DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | 智能体编码(DSpark、Engram UVA 卸载;GPU 待验证) | | | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26([#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)) | 智能体编码 | | | Kimi-K3 | `kimik3` | 2026-07-27 ([#2391](https://github.com/SemiAnalysisAI/InferenceX/pull/2391)) | 智能体编码(仅 DSpark) | 智能体编码非 DSpark 分支(自第 0 天起弃用) | | GLM-5.2 | `glm5.2` | 2026-07-18([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | 智能体编码(非 MTP 分支仍在运行,「仅 MTP」转换仍待执行,见弃用公告) | | diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 5ffe8a29df..6d00f78456 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -22,6 +22,28 @@ INFERENCEX_REPO_ROOT="$( # nothing upstream set it. export PORT="${PORT:-8888}" +# Opt-in for recipes running in the host network namespace. Probe the preferred +# port on the compute node; fall back to an OS-selected port if it is occupied. +# Call immediately before server launch and construct client URLs afterward. +select_available_server_port() { + PORT=$(python3 - "${PORT:-8888}" <<'PYPORT' +import errno +import socket +import sys + +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.bind(("0.0.0.0", int(sys.argv[1]))) + except OSError as exc: + if exc.errno != errno.EADDRINUSE: + raise + sock.bind(("0.0.0.0", 0)) + print(sock.getsockname()[1]) +PYPORT + ) || return $? + export PORT +} + agentic_kv_offload_enabled() { if [[ -z "${KV_OFFLOADING+x}" || -z "$KV_OFFLOADING" ]]; then echo "Error: KV_OFFLOADING must be set for agentic benchmarks" >&2 diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_h200_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_h200_vllm_mtp.sh new file mode 120000 index 0000000000..157d224098 --- /dev/null +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_h200_vllm_mtp.sh @@ -0,0 +1 @@ +dsv41flash_fp4_vllm_mtp.sh \ No newline at end of file diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh new file mode 100644 index 0000000000..f9a6542a14 --- /dev/null +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Native DeepSeek-V4.1-Flash DSpark and Engram UVA weight offload. +# https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash +source "$(dirname "$0")/../../benchmark_lib.sh" +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION +require_agentic_kv_offload_none +export GPU_COUNT="$TP" + +# Complete/resume partial downloads instead of trusting nonempty directories. +if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +nvidia-smi +resolve_trace_source +install_agentic_deps +mkdir -p "$RESULT_DIR" +SERVER_LOG="$RESULT_DIR/server.log" +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_USE_RUST_FRONTEND=1 +export PYTHONUNBUFFERED=1 + +# Preserve the upstream scheduler defaults; size graph capture for the sweep. +NUM_SPEC_TOKENS=5 +CAPTURE_SIZE=1 +while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do + CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) +done + +# Pyxis shares the host network; port 8888 can already belong to a host service. +select_available_server_port +export AIPERF_SERVER_URL="http://localhost:${PORT}" +export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" +echo "Using vLLM endpoint ${AIPERF_SERVER_URL}" + +# Golden AL: golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. +# Accuracy evals keep real block rejection; throughput fixes acceptance to AL 3.51. +if [[ "${EVAL_ONLY:-false}" == true ]]; then + SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":true}' +else + SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}' +fi +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" + --language-model-only + --tokenizer-mode deepseek_v41 + --tool-call-parser deepseek_v41 --enable-auto-tool-choice + --reasoning-parser deepseek_v41 + --engram-config '{"cpu_offload":true}' + --speculative-config "$SPEC_CONFIG" + --max-model-len 1048576 + --max-cudagraph-capture-size "$CAPTURE_SIZE" + --disable-uvicorn-access-log +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [[ "${EVAL_ONLY:-false}" == true ]]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9eeeffd505..fcc0231ea5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -10213,3 +10213,27 @@ glm5.1-fp8-b200-tilert-agentic: dp-attn: false additional-settings: - "DECODE_NODES=1" + +# H200 AgentX arm for DeepSeek-V4.1-Flash. Upstream marks h200 verified and says +# the GB200 NVL4 TP4 layout becomes TP8 on 8-GPU nodes, so this is TP8. +# `precision: fp4` labels the checkpoint's MXFP4 routed experts, as on the +# Blackwell and MI355X arms; Hopper has no FP4 tensor cores, so the MXFP4 +# experts run through the upconverting MoE path. The H100 arm is deferred: the +# sparse attention indexer allocates a [max-num-batched-tokens, max-model-len] +# buffer at startup, which is 16 GiB at 1M context and does not fit 80 GB +# alongside the weights. Tracked separately. +dsv41flash-fp4-h200-vllm-agentic-dspark: + image: vllm/vllm-openai:deepseekv41-flash-0909 + model: deepseek-ai/DeepSeek-V4.1-Flash + model-prefix: dsv41flash + runner: cluster:h200-dgxc + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # 8x141 GB holds the 511 GB checkpoint minus the UVA-offloaded Engram + # tables, so the KV cache stays GPU-resident across the full range. + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 5101335723..95068b5549 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -167,6 +167,53 @@ Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-nego 7. Add script + master entry + launcher routing + changelog together. 8. Run Bash syntax and generation checks. Inspect `spec-decoding`, draft/native method, token count, chat-template use, capture range, and resolved script. +### DeepSeek-V4.1-Flash DSpark on H200 + +`dsv41flash-fp4-h200-vllm-agentic-dspark` is the H200 AgentX arm of the +DeepSeek-V4.1-Flash recipe. It shares `vllm/vllm-openai:deepseekv41-flash-0909` and the +text-only serving script with the Blackwell arms: `deepseek_v41` tokenizer and parsers, +1M context, native five-token DSpark with probabilistic drafting. Throughput uses the [committed golden AL](../golden_al_distribution/dsv41flash_dspark.yaml) of 3.51 for thinking on and five draft tokens, with synthetic rejection sampling and adaptive verification disabled. Accuracy evals retain real block rejection and adaptive verification. + +The arm runs **TP8**, not the upstream TP4. Upstream verifies TP4 on one GB200 NVL4 tray +and states that the same layout becomes TP8 per role on 8-GPU nodes, which is what an +H200 DGXC node is. + +`precision: fp4` labels the checkpoint's MXFP4 routed expert weights, matching the +Blackwell and MI355X arms on the identical checkpoint. Hopper has no FP4 tensor cores, so +those weights run through the upconverting MoE path; the label describes the checkpoint, +not the SKU's native arithmetic. + +`--engram-config '{"cpu_offload":true}'` keeps the Engram tables in pinned host DRAM +reached through UVA, and `kv-offloading: none` describes the separate, GPU-resident KV +cache. Measured on the cluster, the offload moves 11.80 GiB per rank per table for two +tables across 8 ranks — 188.8 GiB — leaving roughly 35.9 GiB per GPU of resident weights +out of 141 GiB. + +Trace corpus: the arm replays the uncapped `semianalysis_cc_traces_weka_062126` corpus, +not the 256k-capped `..._062126_256k` variant, because the model serves 1M context. The +recipe never names a corpus — `resolve_trace_source` picks the uncapped default only +because its `dsv4*` case arm also matches the `dsv41flash` prefix. That is load-bearing +and invisible at the call site, so `runners/test_dsv41flash_h200.py` pins it; narrowing +the arm would silently downgrade this recipe's traces. + +**Why there is no H100 arm.** H100 is not in the upstream hardware table, and the +blocker is not the weights. At 1M context the sparse attention indexer allocates a +`[max-num-batched-tokens, max-model-len]` logits buffer in +`fp8_fp4_paged_mqa_logits`, which at the default 8192 batched tokens is exactly 16 GiB. +That is a fixed startup cost paid during memory profiling, independent of concurrency, so +it fails at concurrency 1 on an 80 GB card even though the resident weights fit. Capping +`--max-num-batched-tokens` shrinks it proportionally; capping `--max-model-len` would too, +but that would force the 256k-capped trace corpus. An H100 arm needs its own script rather +than the shared symlink, and is tracked separately. + +The launcher mounts the repository at `/ix` for this recipe so AgentX runtime directories +are not created under `/workspace`, and it already mounts the shared HF cache, so the +script resolves the model through `HF_HUB_CACHE` rather than a per-node path. The recipe +probes the serving port on the compute node and selects an available one if the preferred +port is occupied; serving, replay, metrics, and eval share that endpoint. + +Source: [upstream recipe](https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml). + ## Validate Run the smallest checks that cover the edited layers. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 20614c18bc..566534b3b0 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -167,6 +167,45 @@ llm-d 不是 srt-slurm 路径:InferenceX 自己持有 Slurm allocation,并 7. 同时添加脚本 + 主配置条目 + launcher 路由 + changelog。 8. 运行 Bash 语法和生成检查;检查 `spec-decoding`、draft/native 方法、token 数、chat-template 使用、capture 范围和解析出的脚本。 +### H200 上的 DeepSeek-V4.1-Flash DSpark + +`dsv41flash-fp4-h200-vllm-agentic-dspark` 是 DeepSeek-V4.1-Flash 配方的 H200 AgentX +分支。它与 Blackwell 分支共用 `vllm/vllm-openai:deepseekv41-flash-0909` 和纯文本服务 +脚本:`deepseek_v41` tokenizer 和解析器、1M 上下文、原生五 token DSpark(概率采样草稿)。吞吐测试使用[已提交的黄金 AL](../golden_al_distribution/dsv41flash_dspark.yaml):thinking 开启、五个草稿 token 对应 3.51,采用合成拒绝采样并关闭自适应验证。准确率 eval 保留真实块拒绝采样和自适应验证。 + +该分支使用 **TP8**,而非上游的 TP4。上游在一个 GB200 NVL4 tray 上验证 TP4,并说明在 +8 GPU 节点上同一布局每个角色变为 TP8,而 H200 DGXC 节点正是 8 GPU 节点。 + +`precision: fp4` 标记检查点中 MXFP4 的路由专家权重,与同一检查点的 Blackwell 和 +MI355X 分支保持一致。Hopper 没有 FP4 tensor core,因此这些权重走上转换的 MoE 路径; +该标签描述检查点,而非 SKU 的原生算力。 + +`--engram-config '{"cpu_offload":true}'` 将 Engram 表放在固定页主机 DRAM 中,通过 UVA +访问;`kv-offloading: none` 描述的是另行驻留 GPU 的 KV cache。集群实测:卸载在 8 个 rank +上为两张表各移出每 rank 11.80 GiB,共 188.8 GiB,使每 GPU 的驻留权重从 141 GiB 中约占 +35.9 GiB。 + +轨迹语料:该分支回放未截断的 `semianalysis_cc_traces_weka_062126` 语料,而不是 256k +截断的 `..._062126_256k` 变体,因为该模型服务 1M 上下文。配方本身并未指定语料 —— +`resolve_trace_source` 选中未截断的默认值,仅仅是因为其 `dsv4*` 分支同时匹配了 +`dsv41flash` 前缀。这一依赖在调用处并不可见却至关重要,因此由 +`runners/test_dsv41flash_h200.py` 固定;收窄该分支会静默地降级本配方的轨迹。 + +**为何没有 H100 分支。** H100 不在上游硬件表中,且瓶颈不在权重。在 1M 上下文下,稀疏 +注意力 indexer 会在 `fp8_fp4_paged_mqa_logits` 中分配一个 +`[max-num-batched-tokens, max-model-len]` 的 logits 缓冲区,在默认 8192 batched tokens +下恰好为 16 GiB。这是显存 profiling 阶段固定支付的启动开销,与并发无关,因此即使驻留 +权重放得下,在 80 GB 卡上并发 1 也会失败。收窄 `--max-num-batched-tokens` 可按比例缩小 +该缓冲区;收窄 `--max-model-len` 同样有效,但会强制使用 256k 截断的轨迹语料。H100 分支 +需要独立脚本而非共享符号链接,另行跟踪。 + +launcher 为该配方将仓库挂载到 `/ix`,避免在 `/workspace` 下创建 AgentX 运行目录;它本来 +就挂载了共享 HF 缓存,因此脚本通过 `HF_HUB_CACHE` 解析模型,而不依赖各节点的独立路径。 +配方在计算节点探测服务端口,首选端口被占用时选择可用端口,服务、回放、指标和 eval 共用 +同一端点。 + +来源:[上游配方](https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml)。 + ## 验证 运行覆盖被修改层的最小检查。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8efc4d5c03..febb824e82 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7146,6 +7146,13 @@ - "The release image ships the same CUDA 13.0.3, FlashInfer 0.6.18 and sgl-kernel 0.4.6.post1 as the nightly. benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps, golden acceptance length 3.39, flashinfer attention with allreduce fusion, fp8 quantization and fp8_e4m3 KV, HiCache kernel IO / page_first layout. TP8/EP1 DRAM HiCache concurrency 2 through 24 unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2966 +- config-keys: + - dsv41flash-fp4-h200-vllm-agentic-dspark + description: + - "Add the H200 AgentX arm for DeepSeek-V4.1-Flash at TP8 with native five-token DSpark and adaptive real verification" + - "Engram tables offload to host DRAM through UVA while the KV cache stays GPU-resident; concurrency 1-128 on the uncapped 062126 trace corpus" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2963 + - config-keys: - glm5.2-fp4-b300-sglang-agentic-mtp scenario-type: diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index d19928fb64..9f882f6e91 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -462,11 +462,17 @@ else BENCH_SCRIPT="${BENCH_BASE}${LEGACY_FW_SUFFIX}${SPEC_SUFFIX}.sh" fi - if [[ "$IMAGE" == *deepseek-v4-hopper* ]]; then + # DeepSeek-V4.1-Flash creates AgentX runtime directories next to the + # repository, which must not land under /workspace. + if [[ "$IMAGE" == *deepseek-v4-hopper* || "$MODEL_PREFIX" == "dsv41flash" ]]; then CONTAINER_MOUNT_DIR=/ix else CONTAINER_MOUNT_DIR=/workspace fi + if [[ "$MODEL_PREFIX" == "dsv41flash" ]]; then + export INFMAX_CONTAINER_WORKSPACE=/ix + export RESULT_DIR=/ix/results + fi srun --jobid=$JOB_ID \ --container-image=$SQUASH_FILE \ diff --git a/runners/test_dsv41flash_h200.py b/runners/test_dsv41flash_h200.py new file mode 100644 index 0000000000..c6818ede79 --- /dev/null +++ b/runners/test_dsv41flash_h200.py @@ -0,0 +1,107 @@ +import json +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh" + +LAUNCH_HARNESS = ''' + salloc() { :; } + squeue() { echo 123; } + srun() { + python3 -c 'import json,os,sys; open(sys.argv[1], "a").write(json.dumps({"args": sys.argv[2:], "result_dir": os.environ.get("RESULT_DIR", "")})+"\\n")' "$SRUN_LOG" "$@" + } + scancel() { :; } + export IS_MULTINODE=false IS_AGENTIC=1 SCENARIO_SUBDIR=agentic/ + export IMAGE=vllm/vllm-openai:deepseekv41-flash-0909 + export HF_HUB_CACHE=/mnt/hf_hub_cache/ RESULT_DIR=/workspace/results + export GITHUB_WORKSPACE="$1" SRUN_LOG="$2" + cd "$GITHUB_WORKSPACE" +''' + + +def launch(sku: str, log: Path, **env: str) -> dict: + exports = " ".join(f"export {key}={value};" for key, value in env.items()) + result = subprocess.run( + [ + "bash", + "-c", + f"{LAUNCH_HARNESS}\n{exports}\nsource runners/launch_{sku}-dgxc-slurm.sh", + "bash", + str(REPO_ROOT), + str(log), + ], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + # The last srun call is the benchmark launch; earlier ones import the image. + return json.loads(log.read_text().splitlines()[-1]) + + +def test_h200_flash_runs_the_vllm_script_from_an_ix_mount(tmp_path: Path) -> None: + sku = "h200" + serve = launch( + sku, + tmp_path / "launch.jsonl", + MODEL_PREFIX="dsv41flash", + MODEL="deepseek-ai/DeepSeek-V4.1-Flash", + PRECISION="fp4", + FRAMEWORK="vllm", + SPEC_DECODING="mtp", + TP="8", + RUNNER_NAME=f"{sku}-test", + EXP_NAME="dsv41flash_tp8_conc1", + ) + script = serve["args"][-1] + assert script == f"benchmarks/single_node/agentic/dsv41flash_fp4_{sku}_vllm_mtp.sh" + assert (REPO_ROOT / script).is_file() + + mounts = next(arg for arg in serve["args"] if arg.startswith("--container-mounts=")) + assert f"{REPO_ROOT}:/ix/," in mounts + assert ":/mnt/hf_hub_cache/," in mounts + # AgentX must not write runtime directories under /workspace. + assert "/workspace" not in mounts + assert serve["result_dir"] == "/ix/results" + assert "--container-workdir=/ix/" in serve["args"] + + +def resolve_loader(model_prefix: str) -> str: + """The public-dataset loader `resolve_trace_source` picks for a prefix.""" + result = subprocess.run( + [ + "bash", + "-c", + # Stub the CLI bootstrap *after* sourcing, so the real one does + # not overwrite the stub, and no dataset is actually downloaded. + 'source "$BENCHMARK_LIB"; ensure_hf_cli() { AIPERF_HF_CLI=true; }; ' + "resolve_trace_source", + ], + env={ + "PATH": "/usr/bin:/bin:/usr/local/bin", + "HOME": str(REPO_ROOT), + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "MODEL_PREFIX": model_prefix, + }, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.split("public-dataset: ")[1].split(" ")[0] + + +def test_flash_replays_the_uncapped_1m_trace_corpus() -> None: + """DeepSeek-V4.1-Flash serves 1M context, so it must not get the 256k corpus. + + This lives with the recipe rather than in the benchmark_lib tests because + the recipe never names a corpus: it inherits one from `resolve_trace_source` + only because the `dsv4*` case arm also matches the `dsv41flash` prefix. + Narrowing that arm would silently downgrade this recipe's traces. + """ + assert resolve_loader("dsv41flash") == "semianalysis_cc_traces_weka_062126" + + +def test_short_context_families_still_get_the_capped_corpus() -> None: + """The uncapped default is context-driven, not a blanket default.""" + assert resolve_loader("qwen3.8next") == "semianalysis_cc_traces_weka_062126_256k" diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 60916cbfa4..dad4914fa6 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -5,6 +5,7 @@ import json import os import signal +import socket import stat import subprocess import sys @@ -3231,3 +3232,27 @@ def test_bfcl_python_preparation_exposes_system_site_packages( assert f"SELECTED_PYTHON=<{python_root / 'venv/bin/python'}>" in result.stdout assert "--prefix" not in result.stdout assert not python_root.exists() + + +@pytest.mark.parametrize("occupied", [False, True]) +def test_select_available_server_port_avoids_an_existing_listener(occupied: bool) -> None: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("0.0.0.0", 0)) + preferred = listener.getsockname()[1] + if occupied: + listener.listen() + else: + listener.close() + result = subprocess.run( + ["bash", "-c", 'source "$BENCHMARK_LIB"; select_available_server_port; ' + 'python3 -c \'import os; print(os.environ["PORT"])\''], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), "PORT": str(preferred)}, + text=True, capture_output=True, check=True, + ) + selected = int(result.stdout.strip()) + if occupied: + assert selected != preferred + else: + assert selected == preferred + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.bind(("0.0.0.0", selected))