From 8d6274611c10af602db70fdcab46a00a400b6e8b Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Tue, 20 Oct 2026 13:41:52 -0700 Subject: [PATCH 1/3] fix: accept native dense KV skip evidence in FPM collection Signed-off-by: Simone Chen --- .../fpm-self-service/implementation.md | 2 + .../collector/fpm_forward/database.py | 14 ++-- .../collector/test_fpm_dense_synthetic_kv.py | 82 +++++++++++++++---- 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index 209a70bb7d..e17e55746e 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -939,6 +939,8 @@ Zero-KV prefill can retain a native `real_prefix` path stamp while its formal se Dense decode can be synthetic by design only when the producer reports enabled warm-up with `warm_eligible: false` and `skip_reason: dense_model_content_insensitive`, and the pinned native evidence corroborates a dense full-attention model and its exact runtime/topology. The supported case is single-rank or attention TP with PP1, DP1, MoE TP1/EP1, and prefill and decode context parallelism both equal to 1. Initialized cache groups must be causal full attention, with no sliding window, attention chunking, convolution or recurrent state; a `FullAttentionSpec` type name alone is insufficient. Matching model and runtime identities and every expected native rank are required. Consistent qualifying points publish as `skip:dense_model_content_insensitive` under the explicit `dense_full_attention_v1` metadata policy. This retains their measured samples and raw provenance; it does not establish real KV content or serving accuracy. Other `fake_fallback` points remain ineligible for direct timing. +Dynamo can report this deliberate dense skip either with the historical `kvwarm_fake_fallback` point stamp or with an explicit per-point `kv_seed_regime: skip:dense_model_content_insensitive` and no fallback stamp. Both formats require the same engine evidence above. A configuration-level skip reason alone does not qualify an unstamped point. Worker-probe sidecars are separate provenance artifacts, not rank timing results, and do not replace the required engine snapshot. + Readiness counts describe the current reaggregation of verified native artifacts. They do not relabel an existing formal pair. Historical publications without the dense classification marker retain their recorded regimes during verification, finalization and validation; any revised publication needs a fresh output tree while preserving the original pair. `ready_for_full_collection` permits the next collection step only for the verified sampled scope. For imported runtime profiles, `runtime_compatibility` separately reports whether the actual collection observations preserve accepted memory; requests without those observations retain the timing-only assessment. Readiness does not establish interpolation coverage, repeatability or workload accuracy. Full execution checks matching usable smoke or formal evidence for every current cell before launching new GPU work; it does not treat a completed memory probe, process exit or existing report file as sufficient. Continue the existing collection-quality, memory-finalization, replay and matched-serving steps afterward. diff --git a/python/aisimulate/collector/fpm_forward/database.py b/python/aisimulate/collector/fpm_forward/database.py index e2eec24064..c4853c292e 100644 --- a/python/aisimulate/collector/fpm_forward/database.py +++ b/python/aisimulate/collector/fpm_forward/database.py @@ -255,9 +255,12 @@ def aggregate_cell( and _dense_synthetic_kv_eligible(plan, cell, cell_dir / "raw", collection) ) - def _kv_seed_regime(point: dict[str, Any], phase: str) -> str: + def _kv_seed_regime(point: dict[str, Any], phase: str, native_regime: str | None) -> str: if phase == "prefill" and (not cell.execution_identity[0] or int(point["total_kv_read_tokens"]) == 0): return "n/a" + # New producers report deliberate skips without a fake-fallback stamp. + if native_regime == "skip:dense_model_content_insensitive": + return native_regime if dense_synthetic else "fake_fallback" if kvwarm_meta is None: return "legacy" skip_reason = kvwarm_meta.get("skip_reason") @@ -283,11 +286,8 @@ def _kv_seed_regime(point: dict[str, Any], phase: str) -> str: batch = int(point["batch_size"]) total_prefill = int(point["total_prefill_tokens"]) total_kv = int(point["total_kv_read_tokens"]) - if ( - cell.execution_identity[0] - and (phase == "decode" or total_kv > 0) - and _kv_seed_regime(point, phase) != "real_kv" - ): + seed_regime = _kv_seed_regime(point, phase, measurement.kv_seed_regime) + if cell.execution_identity[0] and (phase == "decode" or total_kv > 0) and seed_regime != "real_kv": raise ValueError("DeepSeek-V4.1 cached prefill/decode publication requires real_kv measurements") rows.append( { @@ -318,7 +318,7 @@ def _kv_seed_regime(point: dict[str, Any], phase: str) -> str: "total_prefill_tokens": total_prefill, "total_kv_read_tokens": total_kv, "partition_policy": "balanced_v1", - "kv_seed_regime": _kv_seed_regime(point, phase), + "kv_seed_regime": seed_regime, "latency_ms": max(latency for _rank, latency in measurement.rank_wall_times) * 1000.0, "global_warmup_iterations": plan.options.warmup_iterations, "warmup_repeats": 0, diff --git a/python/aisimulate/tests/unit/collector/test_fpm_dense_synthetic_kv.py b/python/aisimulate/tests/unit/collector/test_fpm_dense_synthetic_kv.py index 93e2d30163..c86d11a482 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_dense_synthetic_kv.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_dense_synthetic_kv.py @@ -73,7 +73,16 @@ def _engine(plan, cell, *, version=None, groups=None): } -def _dense_cell(tmp_path, *, groups=None): +def _use_native_dense_skip(payload): + for row in payload["results"]: + row["kv_seed_regime"] = "skip:dense_model_content_insensitive" + for entry in (*payload["results"], *payload["iteration_groups"]): + entry["point"]["sample_reasons"] = [ + reason for reason in entry["point"]["sample_reasons"] if reason != "kvwarm_fake_fallback" + ] + + +def _dense_cell(tmp_path, *, groups=None, native_skip=False): plan, cell, directory = _synthetic_decode_cell( tmp_path, kvwarm=_DENSE_SKIP, @@ -93,6 +102,8 @@ def _dense_cell(tmp_path, *, groups=None): payload["engine"] = _engine(plan, cell, groups=groups) for row in payload["results"]: row["kv_seed_regime"] = "fake_fallback" + if native_skip: + _use_native_dense_skip(payload) path.write_text(json.dumps(payload)) return plan, cell, directory, path @@ -107,15 +118,39 @@ def _regimes(case): [[_FULL], [_FULL, _FULL], [{"type": "UniformTypeKVCacheSpecs", "specs": [_FULL, _FULL]}]], ids=["full", "multiple-full", "wrapped-full"], ) -def test_dense_full_attention_preserves_native_samples_and_becomes_direct_eligible(tmp_path, groups): - case = _dense_cell(tmp_path, groups=groups) +@pytest.mark.parametrize("native_skip", [False, True], ids=["legacy-stamp", "native-skip"]) +def test_dense_full_attention_preserves_native_samples_and_becomes_direct_eligible(tmp_path, groups, native_skip): + case = _dense_cell(tmp_path, groups=groups, native_skip=native_skip) plan, cell, directory, path = case before = path.read_bytes() rows = aggregate_cell(plan, cell, directory, expected_attempt_id="attempt") assert {row["kv_seed_regime"] for row in rows} == {"skip:dense_model_content_insensitive"} assert [row["latency_ms"] for row in rows] == [10.0, 20.0] assert path.read_bytes() == before - assert {row["kv_seed_regime"] for row in json.loads(before)["results"]} == {"fake_fallback"} + assert {row["kv_seed_regime"] for row in json.loads(before)["results"]} == { + "skip:dense_model_content_insensitive" if native_skip else "fake_fallback" + } + + +def test_native_dense_skip_with_separate_worker_probe_uses_startup_engine_evidence(tmp_path): + case = _dense_cell(tmp_path, native_skip=True) + path = case[-1] + payload = json.loads(path.read_text()) + payload["engine"].update(resolution="pending_worker_probe", resolved=None) + path.write_text(json.dumps(payload)) + sidecar = path.with_name("benchmark_merged_worker_probe.json") + sidecar.write_text(json.dumps({"schema": "dynamo.fpm.benchmark_worker_probe", "schema_version": 1})) + before = path.read_bytes(), sidecar.read_bytes() + assert _regimes(case) == {"skip:dense_model_content_insensitive"} + assert (path.read_bytes(), sidecar.read_bytes()) == before + + +def test_native_dense_skip_without_warmup_metadata_is_not_legacy(tmp_path): + case = _dense_cell(tmp_path, native_skip=True) + payload = json.loads(case[-1].read_text()) + del payload["kvwarm"] + case[-1].write_text(json.dumps(payload)) + assert _regimes(case) == {"fake_fallback"} @pytest.mark.parametrize( @@ -141,8 +176,9 @@ def test_dense_full_attention_preserves_native_samples_and_becomes_direct_eligib [{**_FULL, "specs": [{"type": "ShortConvSpec"}]}], ], ) -def test_unknown_or_non_full_attention_cache_never_promotes_dense_skip(tmp_path, groups): - assert _regimes(_dense_cell(tmp_path, groups=groups)) == {"fake_fallback"} +@pytest.mark.parametrize("native_skip", [False, True]) +def test_unknown_or_non_full_attention_cache_never_promotes_dense_skip(tmp_path, groups, native_skip): + assert _regimes(_dense_cell(tmp_path, groups=groups, native_skip=native_skip)) == {"fake_fallback"} @pytest.mark.parametrize( @@ -165,8 +201,9 @@ def test_unknown_or_non_full_attention_cache_never_promotes_dense_skip(tmp_path, ("kv_cache", "groups", None), ], ) -def test_contradictory_or_missing_initialized_identity_stays_fake_fallback(tmp_path, section, key, value): - case = _dense_cell(tmp_path) +@pytest.mark.parametrize("native_skip", [False, True]) +def test_contradictory_or_missing_initialized_identity_stays_fake_fallback(tmp_path, section, key, value, native_skip): + case = _dense_cell(tmp_path, native_skip=native_skip) payload = json.loads(case[-1].read_text()) payload["engine"][section][key] = value case[-1].write_text(json.dumps(payload)) @@ -177,8 +214,9 @@ def test_contradictory_or_missing_initialized_identity_stays_fake_fallback(tmp_p @pytest.mark.parametrize("missing", ["engine", "model", "parallel", "versions", "kv_cache"]) -def test_historical_or_incomplete_engine_evidence_is_not_promoted(tmp_path, missing): - case = _dense_cell(tmp_path) +@pytest.mark.parametrize("native_skip", [False, True]) +def test_historical_or_incomplete_engine_evidence_is_not_promoted(tmp_path, missing, native_skip): + case = _dense_cell(tmp_path, native_skip=native_skip) payload = json.loads(case[-1].read_text()) (payload if missing == "engine" else payload["engine"]).pop(missing) case[-1].write_text(json.dumps(payload)) @@ -263,8 +301,9 @@ def test_historical_saved_identity_retains_dense_evidence_without_reloading_sour {"enabled": True, "warm_eligible": False, "skip_reason": "hybrid_state_layers_unsupported"}, ], ) -def test_full_attention_does_not_convert_failed_or_disabled_warmup(tmp_path, meta): - case = _dense_cell(tmp_path) +@pytest.mark.parametrize("native_skip", [False, True]) +def test_full_attention_does_not_convert_failed_or_disabled_warmup(tmp_path, meta, native_skip): + case = _dense_cell(tmp_path, native_skip=native_skip) payload = json.loads(case[-1].read_text()) payload["kvwarm"] = meta case[-1].write_text(json.dumps(payload)) @@ -283,11 +322,14 @@ def test_dense_skip_with_real_kv_injection_is_rejected(tmp_path): _regimes(case) -def test_unstamped_dense_point_is_not_promoted_from_skip_reason_alone(tmp_path): +@pytest.mark.parametrize("regime", [None, "skip:dataset_unavailable", "not_applicable"]) +def test_unstamped_dense_point_is_not_promoted_from_skip_reason_alone(tmp_path, regime): case = _dense_cell(tmp_path) payload = json.loads(case[-1].read_text()) for row, group in zip(payload["results"], payload["iteration_groups"], strict=True): row.pop("kv_seed_regime") + if regime is not None: + row["kv_seed_regime"] = regime row["point"]["sample_reasons"] = [] group["point"]["sample_reasons"] = [] case[-1].write_text(json.dumps(payload)) @@ -313,10 +355,11 @@ def test_existing_moe_by_construction_and_legacy_contracts_are_unchanged(tmp_pat } -def test_dense_publication_keeps_legacy_classification_until_fresh_publication(tmp_path): +@pytest.mark.parametrize("native_skip", [False, True]) +def test_dense_publication_keeps_legacy_classification_until_fresh_publication(tmp_path, native_skip): import pyarrow.parquet as pq - plan, cell, directory, _path = _dense_cell(tmp_path) + plan, cell, directory, _path = _dense_cell(tmp_path, native_skip=native_skip) legacy_rows = aggregate_cell(plan, cell, directory, expected_attempt_id="attempt", dense_synthetic_kv=False) old_table, old_meta, _ = write_formal_database(plan, legacy_rows, systems_root=tmp_path / "old") old_bytes = old_table.read_bytes(), old_meta.read_bytes() @@ -403,7 +446,8 @@ def test_merged_database_does_not_upgrade_old_dense_cells(tmp_path): @pytest.mark.parametrize("observed", ["full", "convolution", "unreported"]) -def test_public_readiness_uses_corroborated_dense_regime(quality_case, observed): # noqa: F811 +@pytest.mark.parametrize("native_skip", [False, True]) +def test_public_readiness_uses_corroborated_dense_regime(quality_case, observed, native_skip): # noqa: F811 case = quality_case _change_decode(case, fake="all", kvwarm=_DENSE_SKIP) for cell in case["plan"].cells: @@ -411,6 +455,8 @@ def test_public_readiness_uses_corroborated_dense_regime(quality_case, observed) continue for path in (case["campaign"] / "cells" / cell.cell_id / "raw").rglob("benchmark-dp*.json"): payload = json.loads(path.read_text()) + if native_skip: + _use_native_dense_skip(payload) if observed != "unreported": groups = [_FULL] if observed == "full" else [_FULL, {"type": "ShortConvSpec"}] payload["engine"] = _engine(case["plan"], cell, groups=groups) @@ -420,7 +466,9 @@ def test_public_readiness_uses_corroborated_dense_regime(quality_case, observed) report = assess_readiness(case["request"], case["root"], case["checkpoint"].parent) decode = next(row for row in report["selected_cells"] if row["cell"]["workload_kind"] == "decode") assert decode["formal_regime_counts"], decode["blockers"] - assert decode["native_regime_counts"] == {"fake_fallback": 12} + assert decode["native_regime_counts"] == { + "skip:dense_model_content_insensitive" if native_skip else "fake_fallback": 12 + } assert decode["direct_eligible_points"] == (12 if observed == "full" else 0) assert decode["status"] == ("ready" if observed == "full" else "blocked") assert decode["formal_regime_counts"] == { From 954544c6da6e971e2baee3eaabd528c777760546 Mon Sep 17 00:00:00 2001 From: Harrison Saturley-Hall Date: Fri, 13 Nov 2026 12:42:38 -0800 Subject: [PATCH 2/3] ci: switch workflows to renamed AISim runner pools (#861) * ci: switch workflows to renamed AISim runner pools Signed-off-by: Harrison King Saturley-Hall * ci: recognize AISim runner labels in actionlint Signed-off-by: Harrison King Saturley-Hall * ci: use the ARC job-pod workspace for HOME Signed-off-by: Harrison King Saturley-Hall * ci: add containerized canaries for the runner cutover Signed-off-by: Harrison King Saturley-Hall * ci: activate controlled canaries after runner registration Signed-off-by: Harrison King Saturley-Hall * ci: respect the shared runner execution contract Signed-off-by: Harrison King Saturley-Hall --------- Signed-off-by: Harrison King Saturley-Hall (cherry picked from commit 976d75a80649197a68654c102dfb754974fc4c14) Signed-off-by: Harrison King Saturley-Hall --- .github/actionlint.yaml | 5 +- .github/workflows/ci.yml | 50 ++++++------ .github/workflows/collector-check.yml | 8 +- .github/workflows/e2e-accuracy-branch.yml | 14 ++-- .github/workflows/fpe-release-qualify.yml | 4 +- .github/workflows/fpe-support-matrix.yml | 8 +- .github/workflows/fpm-accuracy-branch.yml | 4 +- .github/workflows/nightly-ci.yml | 10 +-- .github/workflows/performance.yml | 4 +- .../workflows/prediction-regression-gate.yml | 4 +- .github/workflows/runner-canary.yml | 80 +++++++++++++++++++ .github/workflows/simulation-performance.yml | 8 +- 12 files changed, 140 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/runner-canary.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index bf8d62f5a1..5bef6bfac2 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -6,5 +6,6 @@ # them. self-hosted-runner: labels: - - prod-aisimulate-default-amd64-v1 - - prod-aisimulate-default-arm64-v1 + - prod-aisim-default-amd64-v1 + - prod-aisim-default-arm64-v1 + - dynamo-cpu-4gi-amd64 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62efd24791..daff665d8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -268,11 +268,11 @@ jobs: matrix: include: - arch: amd64 - runner: prod-aisimulate-default-amd64-v1 + runner: prod-aisim-default-amd64-v1 cargo_deny_target: x86_64-unknown-linux-musl cargo_deny_sha256: 0e8c2aa59128612c90d9e09c02204e912f29a5b8d9a64671b94608cbe09e064f - arch: arm64 - runner: prod-aisimulate-default-arm64-v1 + runner: prod-aisim-default-arm64-v1 cargo_deny_target: aarch64-unknown-linux-musl cargo_deny_sha256: 2b3567a60b7491c159d1cef8b7d8479d1ad2a31e29ef49462634ad4552fcc77d runs-on: ${{ matrix.runner }} @@ -280,7 +280,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -317,11 +317,11 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -336,14 +336,14 @@ jobs: name: Rust feature modes needs: [fast-ci, select-full-ci] if: ${{ needs.select-full-ci.outputs.rust_feature_modes == 'true' }} - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 15 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target-feature-tests container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -382,11 +382,11 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -413,11 +413,11 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -462,11 +462,11 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -601,12 +601,12 @@ jobs: - arch: arm64 # quay.io/pypa/manylinux_2_28_aarch64:2026.08.24-1 container_image: quay.io/pypa/manylinux_2_28_aarch64@sha256:29a6ceb78de5b00494e6e7456a72782a4cb54238de102e7491d15fe47789b4d1 - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 timeout-minutes: 10 container: image: ${{ matrix.container_image }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -644,12 +644,12 @@ jobs: - {suite: cli-build, group: 4, groups: 4} - {suite: support-matrix, group: 1, groups: 1} - {suite: tools-build, group: 1, groups: 1} - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 timeout-minutes: 15 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -765,12 +765,12 @@ jobs: needs.select-full-ci.result == 'success' && needs.select-full-ci.outputs.application_wheel == 'true' }} - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 15 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -787,7 +787,7 @@ jobs: name: Python ${{ matrix.python-version }} compatibility needs: [fast-ci, select-full-ci] if: ${{ needs.select-full-ci.outputs.python_compatibility == 'true' }} - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 30 strategy: fail-fast: false @@ -796,7 +796,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -820,12 +820,12 @@ jobs: name: Engine Golden Regression needs: [fast-ci, select-full-ci] if: ${{ needs.select-full-ci.outputs.engine_golden_regression == 'true' }} - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 45 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -873,11 +873,11 @@ jobs: - arch: arm64 # quay.io/pypa/manylinux_2_28_aarch64:2026.08.24-1 container_image: quay.io/pypa/manylinux_2_28_aarch64@sha256:29a6ceb78de5b00494e6e7456a72782a4cb54238de102e7491d15fe47789b4d1 - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ matrix.container_image }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.github/workflows/collector-check.yml b/.github/workflows/collector-check.yml index f1534d4a9f..943d899434 100644 --- a/.github/workflows/collector-check.yml +++ b/.github/workflows/collector-check.yml @@ -31,12 +31,12 @@ concurrency: jobs: check: name: Check collector data - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 20 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - name: Checkout code @@ -116,13 +116,13 @@ jobs: cross-backend-consistency: name: Perf data sanity (informational) - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 25 continue-on-error: true container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - name: Checkout code diff --git a/.github/workflows/e2e-accuracy-branch.yml b/.github/workflows/e2e-accuracy-branch.yml index e700b2164f..080b46be45 100644 --- a/.github/workflows/e2e-accuracy-branch.yml +++ b/.github/workflows/e2e-accuracy-branch.yml @@ -34,7 +34,7 @@ permissions: jobs: wheel: name: Prepare exact accuracy wheel - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 120 permissions: contents: read @@ -123,11 +123,11 @@ jobs: measurements: name: Prepare shared E2E measurements timeout-minutes: 120 - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -176,13 +176,13 @@ jobs: max-parallel: 4 matrix: shard: [0, 1, 2, 3] - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 360 container: # The runner executes as non-root; use its sudo-enabled CI image. image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner env: OPENBLAS_NUM_THREADS: '1' @@ -286,11 +286,11 @@ jobs: name: Qualify E2E accuracy (${{ inputs.artifact_key }}) needs: [wheel, shards] timeout-minutes: 30 - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.github/workflows/fpe-release-qualify.yml b/.github/workflows/fpe-release-qualify.yml index 3d9a3b7c14..103b14e60d 100644 --- a/.github/workflows/fpe-release-qualify.yml +++ b/.github/workflows/fpe-release-qualify.yml @@ -26,7 +26,7 @@ jobs: prepare: name: Build and identify release FPE wheel if: github.ref == 'refs/heads/main' - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 30 container: # Same pinned manylinux toolchain as the main FPE producer. @@ -74,7 +74,7 @@ jobs: generate: name: Release FPE (${{ matrix.shard.system }}, ${{ matrix.shard.backend }}) needs: prepare - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 480 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} diff --git a/.github/workflows/fpe-support-matrix.yml b/.github/workflows/fpe-support-matrix.yml index a1f503119c..c13bdb758d 100644 --- a/.github/workflows/fpe-support-matrix.yml +++ b/.github/workflows/fpe-support-matrix.yml @@ -35,13 +35,13 @@ env: jobs: prepare-wheel: name: Prepare exact FPE wheel - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 30 container: # quay.io/pypa/manylinux_2_28_x86_64:2026.08.24-1 image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner outputs: wheel-sha256: ${{ steps.digest.outputs.sha256 }} @@ -148,12 +148,12 @@ jobs: generate: name: FPE matrix (${{ matrix.shard.system }}, ${{ matrix.shard.backend }}) needs: [discover-shards, prepare-wheel] - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 480 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner strategy: fail-fast: false diff --git a/.github/workflows/fpm-accuracy-branch.yml b/.github/workflows/fpm-accuracy-branch.yml index 0e8b450e76..a39d6ed728 100644 --- a/.github/workflows/fpm-accuracy-branch.yml +++ b/.github/workflows/fpm-accuracy-branch.yml @@ -29,7 +29,7 @@ permissions: jobs: wheel: name: Prepare exact accuracy wheel - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 120 permissions: contents: read @@ -118,7 +118,7 @@ jobs: campaign: name: Qualify FPM accuracy (${{ inputs.artifact_key }}) needs: wheel - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 360 container: image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 8643875c3c..e56b673052 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -292,12 +292,12 @@ jobs: matrix: arch: [amd64, arm64] python-version: ["3.11", "3.12", "3.13"] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 timeout-minutes: 15 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: # Keep the license policy/tooling at the workflow revision. Older @@ -351,7 +351,7 @@ jobs: matrix: include: - arch: amd64 - runner: prod-aisimulate-default-amd64-v1 + runner: prod-aisim-default-amd64-v1 rust_target: x86_64-unknown-linux-gnu rustup_init_sha256: 4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10 uv_target: x86_64-unknown-linux-gnu @@ -359,7 +359,7 @@ jobs: # quay.io/pypa/manylinux_2_28_x86_64:2026.08.24-1 container_image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb - arch: arm64 - runner: prod-aisimulate-default-arm64-v1 + runner: prod-aisim-default-arm64-v1 rust_target: aarch64-unknown-linux-gnu rustup_init_sha256: 9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792 uv_target: aarch64-unknown-linux-gnu @@ -379,7 +379,7 @@ jobs: # for the runner's node24-based actions. image: ${{ matrix.container_image }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim defaults: run: shell: bash diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 90bf8f07c4..1a52734cda 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -53,7 +53,7 @@ jobs: needs: select if: needs.select.outputs.run_comparison == 'true' name: Forward Prediction Performance (advisory) - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 45 defaults: run: @@ -61,7 +61,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/prediction-regression-gate.yml b/.github/workflows/prediction-regression-gate.yml index c8a0da731e..85d5814111 100644 --- a/.github/workflows/prediction-regression-gate.yml +++ b/.github/workflows/prediction-regression-gate.yml @@ -56,7 +56,7 @@ jobs: collect: name: Collect prediction snapshot (${{ matrix.side }}) needs: refs - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 45 strategy: fail-fast: false @@ -65,7 +65,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - name: Checkout comparison revision diff --git a/.github/workflows/runner-canary.yml b/.github/workflows/runner-canary.yml new file mode 100644 index 0000000000..fa17cbbf64 --- /dev/null +++ b/.github/workflows/runner-canary.yml @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Keep normal producers paused until this smoke matrix passes. +name: Runner canary + +on: + workflow_dispatch: + push: + branches: ["pull-request/*"] + paths: [".github/workflows/runner-canary.yml"] + +permissions: + contents: read + +jobs: + canary: + if: github.repository == 'ai-dynamo/aisim' + name: Runner canary (${{ matrix.pool }}) + strategy: + fail-fast: false + matrix: + include: + - pool: amd64 + runner: prod-aisim-default-amd64-v1 + machine: x86_64 + scratch: true + - pool: arm64 + runner: prod-aisim-default-arm64-v1 + machine: aarch64 + scratch: true + runs-on: ${{ matrix.runner }} + timeout-minutes: 10 + container: + image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} + env: + HOME: /__w/aisim/aisim + USER: runner + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Verify source, job workspace, architecture and storage + shell: bash + env: + EXPECTED_MACHINE: ${{ matrix.machine }} + EXPECT_SCRATCH: ${{ matrix.scratch }} + run: | + set -euo pipefail + test "${GITHUB_REPOSITORY}" = ai-dynamo/aisim + test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" + test "$(pwd -P)" = /__w/aisim/aisim + test "${HOME}" = /__w/aisim/aisim + test -w "${HOME}" + test "$(uname -m)" = "${EXPECTED_MACHINE}" + probe=$(mktemp "${HOME}/.runner-canary.XXXXXX") + rm "${probe}" + if [[ "${EXPECT_SCRATCH}" == true ]]; then + test -w /scratch + probe=$(mktemp /scratch/runner-canary.XXXXXX) + rm "${probe}" + fi + printf 'Verified source %s on %s (%s) with writable HOME=%s\n' "${GITHUB_SHA}" "${RUNNER_NAME}" "${EXPECTED_MACHINE}" "${HOME}" + + shared: + if: github.repository == 'ai-dynamo/aisim' + name: Runner canary (shared) + runs-on: dynamo-cpu-4gi-amd64 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Verify shared runner admission, source and writable workspace + shell: bash + run: | + set -euo pipefail + test "${GITHUB_REPOSITORY}" = ai-dynamo/aisim + test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" + test "$(pwd -P)" = "$(realpath "${GITHUB_WORKSPACE}")" + test "$(uname -m)" = x86_64 + test -w "${HOME}" + probe=$(mktemp .runner-canary.XXXXXX) + rm "${probe}" + printf 'Verified source %s on shared runner %s\n' "${GITHUB_SHA}" "${RUNNER_NAME}" diff --git a/.github/workflows/simulation-performance.yml b/.github/workflows/simulation-performance.yml index fcac6726a7..f73c630e16 100644 --- a/.github/workflows/simulation-performance.yml +++ b/.github/workflows/simulation-performance.yml @@ -165,7 +165,7 @@ jobs: needs: select if: needs.select.outputs.run_comparison == 'true' name: Build simulation wheel (${{ matrix.side }}) - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 30 strategy: fail-fast: false @@ -181,7 +181,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -260,7 +260,7 @@ jobs: needs: [select, build] if: needs.select.outputs.run_comparison == 'true' name: Simulation Performance (advisory) - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 75 defaults: run: @@ -268,7 +268,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner env: BASE_SHA: ${{ needs.select.outputs.base_sha }} From 608244e9c4ee87c5e69f90727f0d0ca316f57baa Mon Sep 17 00:00:00 2001 From: Jason Zhou Date: Sun, 15 Nov 2026 07:39:48 +0000 Subject: [PATCH 3/3] docs: remove trailing whitespace from collector guides --- python/aisim/collector/network/slurm/README.md | 2 +- python/aisim/collector/sglang/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/aisim/collector/network/slurm/README.md b/python/aisim/collector/network/slurm/README.md index 255f746622..6838cdf08e 100755 --- a/python/aisim/collector/network/slurm/README.md +++ b/python/aisim/collector/network/slurm/README.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# 1.nccl-test collection step by step +# 1.nccl-test collection step by step ## 1.1 Replace the "/path/to" in the sctipt with actual path diff --git a/python/aisim/collector/sglang/README.md b/python/aisim/collector/sglang/README.md index 2cbbc3cf68..f125314dd8 100644 --- a/python/aisim/collector/sglang/README.md +++ b/python/aisim/collector/sglang/README.md @@ -156,7 +156,7 @@ SGLANG_LOAD_FORMAT=dummy SGLANG_TEST_NUM_LAYERS=2 \ ``` #### Environment Variables -- `DEEPSEEK_MODEL_PATH`: Path to DeepSeek model +- `DEEPSEEK_MODEL_PATH`: Path to DeepSeek model - `SGLANG_LOAD_FORMAT`: Load format, set to `dummy` to skip weight loading - `SGLANG_TEST_NUM_LAYERS`: Load only specified number of layers (with dummy mode) - `SGLANG_TEST_LAYER`: Layer index to test (default: 0)