From fdd84a8e9377dab2547e171f9ff126769fd51b1f Mon Sep 17 00:00:00 2001 From: PeaBrane Date: Sun, 27 Sep 2026 02:40:48 -0700 Subject: [PATCH 1/3] fix: charge the prefill logits GEMM for MoE-family models MOEModel (Mixtral, gpt-oss, Qwen2/Qwen3-MoE, MiniMax-M2, and the Qwen3-VL-MoE subclass) emitted generation_logits_gemm but no context_logits_gemm. The op was lost when models.py was reformatted in e70c992e; every other family still emits it. Prefill therefore skipped the first token's LM-head projection, and the resident-weight inventory, which is derived from context_ops, left out the LM-head weights. Emit context_logits_gemm with the same shape, bf16 mode, and CP seq_split as the dense and DeepSeek families. Signed-off-by: PeaBrane --- .../src/aisimulate_core/sdk/models/moe.py | 12 ++ .../sdk/models/test_large_ep_model_graphs.py | 8 +- .../unit/sdk/models/test_moe_logits_gemm.py | 113 ++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 python/aisimulate/tests/unit/sdk/models/test_moe_logits_gemm.py diff --git a/python/aisimulate/src/aisimulate_core/sdk/models/moe.py b/python/aisimulate/src/aisimulate_core/sdk/models/moe.py index c2f269ac9c..b73f123885 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/models/moe.py +++ b/python/aisimulate/src/aisimulate_core/sdk/models/moe.py @@ -316,6 +316,18 @@ def __init__(self, topk: int, num_experts: int, moe_inter_size: int, *args, back ) ) # logits gemm + self.context_ops.extend( + [ + ops.GEMM( + "context_logits_gemm", + 1, + self._vocab_size // tp_size, + h, + common.GEMMQuantMode.bfloat16, + seq_split=cp, + ) + ] + ) self.generation_ops.extend( [ ops.GEMM( diff --git a/python/aisimulate/tests/unit/sdk/models/test_large_ep_model_graphs.py b/python/aisimulate/tests/unit/sdk/models/test_large_ep_model_graphs.py index 38e21c5731..0bb85f5b31 100644 --- a/python/aisimulate/tests/unit/sdk/models/test_large_ep_model_graphs.py +++ b/python/aisimulate/tests/unit/sdk/models/test_large_ep_model_graphs.py @@ -666,7 +666,9 @@ def test_sglang_uses_the_model_wide_gemm_mode(self): class TestMOEModelLargeEP: # RECORDED from SGLangEPMOEModel @ 8372e60 under the A6 mapping. The legacy # large-EP graph replicates the embedding (no vocab//tp shard) and ends at - # the logits gemm — no ``{p}_embedding_ar``, no P2P. + # the logits gemm — no ``{p}_embedding_ar``, no P2P. The legacy context + # graph also lacked the prefill logits gemm (inherited from the fused + # MOEModel); prefill now projects its last token to the vocabulary too. CONTEXT: ClassVar[list[str]] = [ "context_embedding", "context_add_norm_1", @@ -678,6 +680,7 @@ class TestMOEModelLargeEP: "context_moe_dispatch", "context_moe", "context_moe_combine", + "context_logits_gemm", ] GENERATION: ClassVar[list[str]] = [ "generation_embedding", @@ -915,6 +918,9 @@ def test_moe_family_fused(self): "context_moe_pre_dispatch", "context_moe", "context_moe_post_dispatch", + # Not in the 8372e60 recording: the family had dropped its + # prefill logits gemm; every other family emits one. + "context_logits_gemm", "context_embedding_ar", "context_p2p", ] diff --git a/python/aisimulate/tests/unit/sdk/models/test_moe_logits_gemm.py b/python/aisimulate/tests/unit/sdk/models/test_moe_logits_gemm.py new file mode 100644 index 0000000000..5add8aa42d --- /dev/null +++ b/python/aisimulate/tests/unit/sdk/models/test_moe_logits_gemm.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""MoE-family prefill pays for the vocabulary projection, as every family does. + +Prefill emits the first output token, so its forward pass ends with the same +vocabulary-sharded LM-head GEMM that each decode step runs. The resident +weight inventory is also derived from the context graph, so a prefill graph +without that GEMM under-counts both prefill latency and LM-head weights. +""" + +from __future__ import annotations + +import pytest + +from aisimulate_core.sdk import common, config +from aisimulate_core.sdk.models import get_model + +pytestmark = pytest.mark.unit + +_BF16 = { + "gemm_quant_mode": common.GEMMQuantMode.bfloat16, + "moe_quant_mode": common.MoEQuantMode.bfloat16, + "kvcache_quant_mode": common.KVCacheQuantMode.bfloat16, + "fmha_quant_mode": common.FMHAQuantMode.bfloat16, +} + + +def _vocab_heads(op_list): + return [ + (op._name, op._n, op._k, op._quant_mode, op._scale_factor) for op in op_list if op._name.endswith("logits_gemm") + ] + + +@pytest.mark.parametrize("backend", ["trtllm", "vllm", "sglang"]) +def test_moe_vocab_head_matches_the_dense_family(backend): + # Qwen3-235B-A22B (MOE family) and Qwen3-8B (LLAMA family) share + # hidden_size 4096 and the untied 151936-token vocabulary, so their + # LM-head work must be identical in both phases. + moe = get_model( + "Qwen/Qwen3-235B-A22B", + config.ModelConfig(tp_size=2, moe_tp_size=1, moe_ep_size=2, **_BF16), + backend, + ) + dense = get_model("Qwen/Qwen3-8B", config.ModelConfig(tp_size=2, **_BF16), backend) + + assert _vocab_heads(dense.context_ops) == [ + ("context_logits_gemm", 151936 // 2, 4096, common.GEMMQuantMode.bfloat16, 1.0) + ] + assert _vocab_heads(moe.context_ops) == _vocab_heads(dense.context_ops) + assert _vocab_heads(moe.generation_ops) == _vocab_heads(dense.generation_ops) + + +@pytest.mark.parametrize( + ("model_path", "backend", "parallel"), + [ + pytest.param( + "openai/gpt-oss-20b", + "trtllm", + {"tp_size": 2, "moe_tp_size": 1, "moe_ep_size": 2}, + id="gpt-oss-trtllm", + ), + pytest.param( + "mistralai/Mixtral-8x7B-v0.1", + "vllm", + {"tp_size": 2, "moe_tp_size": 2, "moe_ep_size": 1}, + id="mixtral-vllm", + ), + pytest.param( + "MiniMaxAI/MiniMax-M2.5", + "vllm", + {"tp_size": 4, "pp_size": 2, "moe_tp_size": 1, "moe_ep_size": 4}, + id="minimax-m2-vllm-pp2", + ), + pytest.param( + "Qwen/Qwen3-VL-30B-A3B-Instruct", + "vllm", + {"tp_size": 2, "moe_tp_size": 1, "moe_ep_size": 2}, + id="qwen3-vl-moe-vllm", + ), + pytest.param( + "Qwen/Qwen3-30B-A3B", + "sglang", + {"cp_size": 2, "cp_style": "allgather", "moe_tp_size": 1, "moe_ep_size": 2}, + id="qwen3-moe-sglang-cp2", + ), + pytest.param( + "Qwen/Qwen3-235B-A22B", + "sglang", + { + "attention_dp_size": 8, + "moe_tp_size": 1, + "moe_ep_size": 8, + "moe_backend": "deepep_moe", + "moe_comm_backend": {"context": "deepep_ht", "generation": "deepep_ll"}, + "num_gpus_per_node": 8, + }, + id="qwen3-moe-sglang-large-ep", + ), + ], +) +def test_moe_prefill_projects_to_the_decode_vocab_shard(model_path, backend, parallel): + model = get_model(model_path, config.ModelConfig(**_BF16, **parallel), backend) + + context_heads = _vocab_heads(model.context_ops) + generation_heads = _vocab_heads(model.generation_ops) + assert len(context_heads) == len(generation_heads) == 1 + (_, *context_shape, context_scale) = context_heads[0] + (_, *generation_shape, _) = generation_heads[0] + # Same vocab shard, hidden width and bf16 LM head as decode, charged once + # per prefill forward pass (not per layer). + assert context_shape == generation_shape + assert context_scale == 1.0 From 1a743096cd93a82ee91700589f3c8c7b325df1e4 Mon Sep 17 00:00:00 2001 From: PeaBrane Date: Sun, 27 Sep 2026 02:48:19 -0700 Subject: [PATCH 2/3] test: pin engine goldens for the MoE prefill logits GEMM Refresh the records that the prefill logits GEMM moves, following the selective pin_goldens.py --refresh workflow. Six per-op cases each gain one context_logits_gemm entry of 0.0265 to 0.0275 ms. All other per-op values are unchanged. Twenty-one engine-step records from the 1e-4 tolerance Qwen3-235B-A22B and MiniMax-M2.5 hybrid, empirical, and SOL cases increase their prefill-bearing metrics by 0.020 to 0.084 ms. Decode-only metrics are unchanged. Signed-off-by: PeaBrane --- crates/core/parity_tests/perfmodel/README.md | 17 +++ .../perfmodel/goldens/engine_step.json | 126 +++++++++--------- .../perfmodel/goldens/per_op.json | 42 +++++- 3 files changed, 116 insertions(+), 69 deletions(-) diff --git a/crates/core/parity_tests/perfmodel/README.md b/crates/core/parity_tests/perfmodel/README.md index 5b543c9f3b..46fe589871 100644 --- a/crates/core/parity_tests/perfmodel/README.md +++ b/crates/core/parity_tests/perfmodel/README.md @@ -121,6 +121,23 @@ using `pin_goldens.py --refresh`. Each refreshed record retains its source commit in `post_freeze_pins`; test matrices and tolerances are unchanged. These are prediction-regression baselines, not whole-model silicon validation. +### MoE-family prefill logits GEMM + +`MOEModel` (Mixtral, GPT-OSS, Qwen2/Qwen3-MoE, MiniMax-M2) now emits +`context_logits_gemm`, the prefill LM-head projection every other family +already had. The selective refresh from +`fdd84a8e9377dab2547e171f9ff126769fd51b1f` covers 21 engine-step records and +six per-op cases. No compiled-engine reference moved outside its tolerance. + +Each per-op case gains exactly one `context_logits_gemm` entry of 0.0265 to +0.0275 ms. All other per-op latencies, energies, and source labels are +unchanged. The engine-step records are the Qwen3-235B-A22B and MiniMax-M2.5 +hybrid, empirical, and SOL cases, which use a 1e-4 relative tolerance. Their +prefill-bearing metrics increase by 0.020 to 0.084 ms. Decode-only metrics are +unchanged. MoE records compared at the 1% tolerance still pass and were not +refreshed. The refreshed records are exactly the `post_freeze_pins` entries +that name the commit above, each passed to `pin_goldens.py --refresh`. + ## Engine-Step Benchmark Historical Python-vs-Rust speedup numbers (dated + commit-stamped) live in diff --git a/crates/core/parity_tests/perfmodel/goldens/engine_step.json b/crates/core/parity_tests/perfmodel/goldens/engine_step.json index 848ff04bee..19fd4d69e7 100644 --- a/crates/core/parity_tests/perfmodel/goldens/engine_step.json +++ b/crates/core/parity_tests/perfmodel/goldens/engine_step.json @@ -648,28 +648,28 @@ "minimax-m25-b200-vllm-empirical": { "agg": { "values": { - "agg_request": 107.93990632676808, - "agg_tpot": 24.53642650671439, - "agg_ttft": 83.40347982005369 + "agg_request": 107.9952396903705, + "agg_tpot": 24.562465736644942, + "agg_ttft": 83.43277395372556 } }, "disagg": { "values": { - "disagg_request": 51.654, + "disagg_request": 51.704, "disagg_tpot": 7.696, - "disagg_ttft": 43.958 + "disagg_ttft": 44.008 } }, "mixed": { "values": { - "mixed_step": 24.536426506714385 + "mixed_step": 24.562465736644942 } }, "static": { "values": { - "static_ctx": 22.201, + "static_ctx": 22.227, "static_gen": 8.172, - "static_total": 30.373 + "static_total": 30.399 } } }, @@ -774,16 +774,16 @@ "minimax-m25-nvfp4-h200-vllm-hybrid-xprofile": { "agg": { "values": { - "agg_request": 141.41954389326563, - "agg_tpot": 40.29155006741912, - "agg_ttft": 101.1279938258465 + "agg_request": 141.50392052989315, + "agg_tpot": 40.33125671994972, + "agg_ttft": 101.17266380994343 } }, "disagg": { "values": { - "disagg_request": 85.957, + "disagg_request": 86.035, "disagg_tpot": 11.567, - "disagg_ttft": 74.39 + "disagg_ttft": 74.468 } }, "mixed": { @@ -791,9 +791,9 @@ }, "static": { "values": { - "static_ctx": 37.571, + "static_ctx": 37.61, "static_gen": 280.442, - "static_total": 318.01300000000003 + "static_total": 318.052 } } }, @@ -1171,98 +1171,98 @@ "qwen3-235b-a22b-b200-vllm-empirical": { "agg": { "values": { - "agg_request": 159.98137995897213, - "agg_tpot": 35.47359056892807, - "agg_ttft": 124.50778939004407 + "agg_request": 160.03787657016372, + "agg_tpot": 35.500177209488804, + "agg_ttft": 124.5376993606749 } }, "disagg": { "values": { - "disagg_request": 74.885, + "disagg_request": 74.938, "disagg_tpot": 11.763, - "disagg_ttft": 63.122 + "disagg_ttft": 63.175 } }, "mixed": { "values": { - "mixed_step": 35.47359056892807 + "mixed_step": 35.500177209488804 } }, "static": { "values": { - "static_ctx": 31.88, + "static_ctx": 31.907, "static_gen": 9.061, - "static_total": 40.941 + "static_total": 40.968 } } }, "qwen3-235b-a22b-b200-vllm-hybrid-xprofile": { "agg": { "values": { - "agg_request": 139.61237092888715, - "agg_tpot": 25.888174554770423, - "agg_ttft": 113.72419637411673 + "agg_request": 139.66880438593182, + "agg_tpot": 25.91473147573262, + "agg_ttft": 113.7540729101992 } }, "disagg": { "values": { - "disagg_request": 54.863, + "disagg_request": 54.915, "disagg_tpot": 10.837, - "disagg_ttft": 44.026 + "disagg_ttft": 44.078 } }, "mixed": { "values": { - "mixed_step": 25.888174554770423 + "mixed_step": 25.91473147573262 } }, "static": { "values": { - "static_ctx": 22.236, + "static_ctx": 22.262, "static_gen": 335.478, - "static_total": 357.714 + "static_total": 357.74 } } }, "qwen3-235b-a22b-b200-vllm-hybrid-xquant": { "agg": { "values": { - "agg_request": 285.5999684263735, - "agg_tpot": 94.58822043594047, - "agg_ttft": 191.01174799043304 + "agg_request": 285.65640188341814, + "agg_tpot": 94.61477735690265, + "agg_ttft": 191.04162452651548 } }, "disagg": { "values": { - "disagg_request": 198.068, + "disagg_request": 198.12, "disagg_tpot": 17.893, - "disagg_ttft": 180.175 + "disagg_ttft": 180.227 } }, "mixed": { "values": { - "mixed_step": 94.58822043594046 + "mixed_step": 94.61477735690265 } }, "static": { "values": { - "static_ctx": 90.997, + "static_ctx": 91.024, "static_gen": 13.615, - "static_total": 104.612 + "static_total": 104.639 } } }, "qwen3-235b-a22b-b200-vllm-sol": { "mixed": { "values": { - "mixed_step": 12.623676833141841 + "mixed_step": 12.644012102544439 } }, "static": { "values": { - "static_ctx": 12.612, + "static_ctx": 12.632, "static_gen": 0.735, - "static_total": 13.347 + "static_total": 13.366999999999999 } } }, @@ -2231,19 +2231,19 @@ }, "minimax-m25-b200-vllm-empirical::agg": { "engine": "rust", - "git_head": "0a51476b6ab90bc0e475bd41d4b1c7abbef07b95" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m25-b200-vllm-empirical::disagg": { "engine": "rust", - "git_head": "0a51476b6ab90bc0e475bd41d4b1c7abbef07b95" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m25-b200-vllm-empirical::mixed": { "engine": "rust", - "git_head": "0a51476b6ab90bc0e475bd41d4b1c7abbef07b95" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m25-b200-vllm-empirical::static": { "engine": "rust", - "git_head": "0a51476b6ab90bc0e475bd41d4b1c7abbef07b95" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m25-b200-vllm-isl1024-osl2::agg": { "engine": "rust", @@ -2311,11 +2311,11 @@ }, "minimax-m25-nvfp4-h200-vllm-hybrid-xprofile::agg": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m25-nvfp4-h200-vllm-hybrid-xprofile::disagg": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m25-nvfp4-h200-vllm-hybrid-xprofile::mixed": { "engine": "rust", @@ -2323,7 +2323,7 @@ }, "minimax-m25-nvfp4-h200-vllm-hybrid-xprofile::static": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m27-b200-vllm-isl1024-osl2::agg": { "engine": "rust", @@ -2539,59 +2539,59 @@ }, "qwen3-235b-a22b-b200-vllm-empirical::agg": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-empirical::disagg": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-empirical::mixed": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-empirical::static": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-hybrid-xprofile::agg": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-hybrid-xprofile::disagg": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-hybrid-xprofile::mixed": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-hybrid-xprofile::static": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-hybrid-xquant::agg": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-hybrid-xquant::disagg": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-hybrid-xquant::mixed": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-hybrid-xquant::static": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-sol::mixed": { "engine": "rust", - "git_head": "760b6c92edc00389ff79938b62b028c01bcfefdd" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-a22b-b200-vllm-sol::static": { "engine": "rust", - "git_head": "760b6c92edc00389ff79938b62b028c01bcfefdd" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "qwen3-235b-b200-sglang-adp8-etp4ep2::agg": { "engine": "rust", diff --git a/crates/core/parity_tests/perfmodel/goldens/per_op.json b/crates/core/parity_tests/perfmodel/goldens/per_op.json index 2497ed6067..e4ec132044 100644 --- a/crates/core/parity_tests/perfmodel/goldens/per_op.json +++ b/crates/core/parity_tests/perfmodel/goldens/per_op.json @@ -27,6 +27,11 @@ "latency_ms": 0.04589320056140423, "source": "silicon" }, + "context_logits_gemm": { + "energy_wms": 24.87529863166794, + "latency_ms": 0.027535857502495716, + "source": "silicon" + }, "context_moe": { "energy_wms": 4650.878357851409, "latency_ms": 7.353446388244628, @@ -380,6 +385,11 @@ "latency_ms": 0.03029088020324707, "source": "silicon" }, + "context_logits_gemm": { + "energy_wms": 0.0, + "latency_ms": 0.027021467449039823, + "source": "silicon" + }, "context_moe": { "energy_wms": 0.0, "latency_ms": 14.602319259643556, @@ -511,6 +521,11 @@ "latency_ms": 0.03277568101882934, "source": "silicon" }, + "context_logits_gemm": { + "energy_wms": 0.0, + "latency_ms": 0.026514363978156354, + "source": "silicon" + }, "context_moe": { "energy_wms": 0.0, "latency_ms": 10.724710392951964, @@ -642,6 +657,11 @@ "latency_ms": 0.0782525783777237, "source": "silicon" }, + "context_logits_gemm": { + "energy_wms": 0.0, + "latency_ms": 0.026942310940892625, + "source": "silicon" + }, "context_moe": { "energy_wms": 0.0, "latency_ms": 27.531174778938293, @@ -773,6 +793,11 @@ "latency_ms": 0.03277568101882934, "source": "silicon" }, + "context_logits_gemm": { + "energy_wms": 0.0, + "latency_ms": 0.026514363978156354, + "source": "silicon" + }, "context_moe": { "energy_wms": 0.0, "latency_ms": 10.724710392951964, @@ -1040,6 +1065,11 @@ "latency_ms": 0.029959681034088137, "source": "silicon" }, + "context_logits_gemm": { + "energy_wms": 0.0, + "latency_ms": 0.02717842079981671, + "source": "silicon" + }, "context_moe": { "energy_wms": 0.0, "latency_ms": 3.73831672668457, @@ -1162,7 +1192,7 @@ "post_freeze_pins": { "gpt-oss-20b-b200-trtllm-isl1024-osl2": { "engine": "rust", - "git_head": "36dcc8f3afe9e6e2e9de976737b6337fad8c4d74" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "kimi-k25-b200-sglang-isl1024-osl2": { "engine": "rust", @@ -1174,19 +1204,19 @@ }, "minimax-m25-b200-sglang-isl1024-osl2": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m25-b200-vllm-isl1024-osl2": { "engine": "rust", - "git_head": "0a51476b6ab90bc0e475bd41d4b1c7abbef07b95" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m25-b200-vllm-sampled-prefix": { "engine": "rust", - "git_head": "0a51476b6ab90bc0e475bd41d4b1c7abbef07b95" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "minimax-m27-b200-vllm-isl1024-osl2": { "engine": "rust", - "git_head": "0a51476b6ab90bc0e475bd41d4b1c7abbef07b95" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" }, "nemotron-nas-b200-trtllm-isl1024-osl2": { "engine": "rust", @@ -1194,7 +1224,7 @@ }, "qwen3-30b-a3b-b200-vllm-isl1024-osl2": { "engine": "rust", - "git_head": "eb4f61cb5d11f17898d7a0f1fd47def268445a6b" + "git_head": "fdd84a8e9377dab2547e171f9ff126769fd51b1f" } } } From 812ba0c75556319f0f4a375eed9b64aa7d67dd37 Mon Sep 17 00:00:00 2001 From: Harrison Saturley-Hall Date: Fri, 13 Nov 2026 12:42:38 -0800 Subject: [PATCH 3/3] ci: switch workflows to renamed AISim runner pools (#861) * ci: switch workflows to renamed AISim runner pools Signed-off-by: Harrison King Saturley-Hall * ci: recognize AISim runner labels in actionlint Signed-off-by: Harrison King Saturley-Hall * ci: use the ARC job-pod workspace for HOME Signed-off-by: Harrison King Saturley-Hall * ci: add containerized canaries for the runner cutover Signed-off-by: Harrison King Saturley-Hall * ci: activate controlled canaries after runner registration Signed-off-by: Harrison King Saturley-Hall * ci: respect the shared runner execution contract Signed-off-by: Harrison King Saturley-Hall --------- Signed-off-by: Harrison King Saturley-Hall (cherry picked from commit 976d75a80649197a68654c102dfb754974fc4c14) Signed-off-by: Harrison King Saturley-Hall --- .github/actionlint.yaml | 5 +- .github/workflows/ci.yml | 50 ++++++------ .github/workflows/collector-check.yml | 8 +- .github/workflows/e2e-accuracy-branch.yml | 14 ++-- .github/workflows/fpe-release-qualify.yml | 4 +- .github/workflows/fpe-support-matrix.yml | 8 +- .github/workflows/fpm-accuracy-branch.yml | 4 +- .github/workflows/nightly-ci.yml | 10 +-- .github/workflows/performance.yml | 4 +- .../workflows/prediction-regression-gate.yml | 4 +- .github/workflows/runner-canary.yml | 80 +++++++++++++++++++ .github/workflows/simulation-performance.yml | 8 +- 12 files changed, 140 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/runner-canary.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index bf8d62f5a1..5bef6bfac2 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -6,5 +6,6 @@ # them. self-hosted-runner: labels: - - prod-aisimulate-default-amd64-v1 - - prod-aisimulate-default-arm64-v1 + - prod-aisim-default-amd64-v1 + - prod-aisim-default-arm64-v1 + - dynamo-cpu-4gi-amd64 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61fd34211a..5f6e6c78cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -268,11 +268,11 @@ jobs: matrix: include: - arch: amd64 - runner: prod-aisimulate-default-amd64-v1 + runner: prod-aisim-default-amd64-v1 cargo_deny_target: x86_64-unknown-linux-musl cargo_deny_sha256: 0e8c2aa59128612c90d9e09c02204e912f29a5b8d9a64671b94608cbe09e064f - arch: arm64 - runner: prod-aisimulate-default-arm64-v1 + runner: prod-aisim-default-arm64-v1 cargo_deny_target: aarch64-unknown-linux-musl cargo_deny_sha256: 2b3567a60b7491c159d1cef8b7d8479d1ad2a31e29ef49462634ad4552fcc77d runs-on: ${{ matrix.runner }} @@ -280,7 +280,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -317,11 +317,11 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -336,14 +336,14 @@ jobs: name: Rust feature modes needs: [fast-ci, select-full-ci] if: ${{ needs.select-full-ci.outputs.rust_feature_modes == 'true' }} - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 15 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target-feature-tests container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -382,11 +382,11 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -413,11 +413,11 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -462,11 +462,11 @@ jobs: fail-fast: false matrix: arch: [amd64, arm64] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -601,12 +601,12 @@ jobs: - arch: arm64 # quay.io/pypa/manylinux_2_28_aarch64:2026.08.24-1 container_image: quay.io/pypa/manylinux_2_28_aarch64@sha256:29a6ceb78de5b00494e6e7456a72782a4cb54238de102e7491d15fe47789b4d1 - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 timeout-minutes: 10 container: image: ${{ matrix.container_image }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -644,12 +644,12 @@ jobs: - {suite: cli-build, group: 4, groups: 4} - {suite: support-matrix, group: 1, groups: 1} - {suite: tools-build, group: 1, groups: 1} - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 timeout-minutes: 15 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -765,12 +765,12 @@ jobs: needs.select-full-ci.result == 'success' && needs.select-full-ci.outputs.application_wheel == 'true' }} - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 15 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -787,7 +787,7 @@ jobs: name: Python ${{ matrix.python-version }} compatibility needs: [fast-ci, select-full-ci] if: ${{ needs.select-full-ci.outputs.python_compatibility == 'true' }} - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 30 strategy: fail-fast: false @@ -796,7 +796,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -820,12 +820,12 @@ jobs: name: Engine Golden Regression needs: [fast-ci, select-full-ci] if: ${{ needs.select-full-ci.outputs.engine_golden_regression == 'true' }} - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 45 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -873,11 +873,11 @@ jobs: - arch: arm64 # quay.io/pypa/manylinux_2_28_aarch64:2026.08.24-1 container_image: quay.io/pypa/manylinux_2_28_aarch64@sha256:29a6ceb78de5b00494e6e7456a72782a4cb54238de102e7491d15fe47789b4d1 - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 container: image: ${{ matrix.container_image }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.github/workflows/collector-check.yml b/.github/workflows/collector-check.yml index f1534d4a9f..943d899434 100644 --- a/.github/workflows/collector-check.yml +++ b/.github/workflows/collector-check.yml @@ -31,12 +31,12 @@ concurrency: jobs: check: name: Check collector data - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 20 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - name: Checkout code @@ -116,13 +116,13 @@ jobs: cross-backend-consistency: name: Perf data sanity (informational) - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 25 continue-on-error: true container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - name: Checkout code diff --git a/.github/workflows/e2e-accuracy-branch.yml b/.github/workflows/e2e-accuracy-branch.yml index e700b2164f..080b46be45 100644 --- a/.github/workflows/e2e-accuracy-branch.yml +++ b/.github/workflows/e2e-accuracy-branch.yml @@ -34,7 +34,7 @@ permissions: jobs: wheel: name: Prepare exact accuracy wheel - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 120 permissions: contents: read @@ -123,11 +123,11 @@ jobs: measurements: name: Prepare shared E2E measurements timeout-minutes: 120 - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -176,13 +176,13 @@ jobs: max-parallel: 4 matrix: shard: [0, 1, 2, 3] - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 360 container: # The runner executes as non-root; use its sudo-enabled CI image. image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner env: OPENBLAS_NUM_THREADS: '1' @@ -286,11 +286,11 @@ jobs: name: Qualify E2E accuracy (${{ inputs.artifact_key }}) needs: [wheel, shards] timeout-minutes: 30 - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.github/workflows/fpe-release-qualify.yml b/.github/workflows/fpe-release-qualify.yml index 9b7e4d4041..dcf608ffe3 100644 --- a/.github/workflows/fpe-release-qualify.yml +++ b/.github/workflows/fpe-release-qualify.yml @@ -26,7 +26,7 @@ jobs: prepare: name: Build and identify release FPE wheel if: github.ref == 'refs/heads/main' - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 30 container: # Same pinned manylinux toolchain as the main FPE producer. @@ -74,7 +74,7 @@ jobs: generate: name: Release FPE (${{ matrix.shard.system }}, ${{ matrix.shard.backend }}) needs: prepare - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 480 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} diff --git a/.github/workflows/fpe-support-matrix.yml b/.github/workflows/fpe-support-matrix.yml index 782d439cfb..82826f323d 100644 --- a/.github/workflows/fpe-support-matrix.yml +++ b/.github/workflows/fpe-support-matrix.yml @@ -35,13 +35,13 @@ env: jobs: prepare-wheel: name: Prepare exact FPE wheel - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 30 container: # quay.io/pypa/manylinux_2_28_x86_64:2026.08.24-1 image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner outputs: wheel-sha256: ${{ steps.digest.outputs.sha256 }} @@ -148,12 +148,12 @@ jobs: generate: name: FPE matrix (${{ matrix.shard.system }}, ${{ matrix.shard.backend }}) needs: [discover-shards, prepare-wheel] - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 480 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner strategy: fail-fast: false diff --git a/.github/workflows/fpm-accuracy-branch.yml b/.github/workflows/fpm-accuracy-branch.yml index 0e8b450e76..a39d6ed728 100644 --- a/.github/workflows/fpm-accuracy-branch.yml +++ b/.github/workflows/fpm-accuracy-branch.yml @@ -29,7 +29,7 @@ permissions: jobs: wheel: name: Prepare exact accuracy wheel - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 120 permissions: contents: read @@ -118,7 +118,7 @@ jobs: campaign: name: Qualify FPM accuracy (${{ inputs.artifact_key }}) needs: wheel - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 360 container: image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index c6534fca08..864f6b13bb 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -292,12 +292,12 @@ jobs: matrix: arch: [amd64, arm64] python-version: ["3.11", "3.12", "3.13"] - runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 + runs-on: prod-aisim-default-${{ matrix.arch }}-v1 timeout-minutes: 15 container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: # Keep the license policy/tooling at the workflow revision. Older @@ -351,7 +351,7 @@ jobs: matrix: include: - arch: amd64 - runner: prod-aisimulate-default-amd64-v1 + runner: prod-aisim-default-amd64-v1 rust_target: x86_64-unknown-linux-gnu rustup_init_sha256: 4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10 uv_target: x86_64-unknown-linux-gnu @@ -359,7 +359,7 @@ jobs: # quay.io/pypa/manylinux_2_28_x86_64:2026.08.24-1 container_image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb - arch: arm64 - runner: prod-aisimulate-default-arm64-v1 + runner: prod-aisim-default-arm64-v1 rust_target: aarch64-unknown-linux-gnu rustup_init_sha256: 9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792 uv_target: aarch64-unknown-linux-gnu @@ -379,7 +379,7 @@ jobs: # for the runner's node24-based actions. image: ${{ matrix.container_image }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim defaults: run: shell: bash diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 5eab15de23..f40711142b 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -53,7 +53,7 @@ jobs: needs: select if: needs.select.outputs.run_comparison == 'true' name: Forward Prediction Performance (advisory) - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 45 defaults: run: @@ -61,7 +61,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/prediction-regression-gate.yml b/.github/workflows/prediction-regression-gate.yml index 5835426e94..35f31ccaa3 100644 --- a/.github/workflows/prediction-regression-gate.yml +++ b/.github/workflows/prediction-regression-gate.yml @@ -56,7 +56,7 @@ jobs: collect: name: Collect prediction snapshot (${{ matrix.side }}) needs: refs - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 45 strategy: fail-fast: false @@ -65,7 +65,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - name: Checkout comparison revision diff --git a/.github/workflows/runner-canary.yml b/.github/workflows/runner-canary.yml new file mode 100644 index 0000000000..fa17cbbf64 --- /dev/null +++ b/.github/workflows/runner-canary.yml @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Keep normal producers paused until this smoke matrix passes. +name: Runner canary + +on: + workflow_dispatch: + push: + branches: ["pull-request/*"] + paths: [".github/workflows/runner-canary.yml"] + +permissions: + contents: read + +jobs: + canary: + if: github.repository == 'ai-dynamo/aisim' + name: Runner canary (${{ matrix.pool }}) + strategy: + fail-fast: false + matrix: + include: + - pool: amd64 + runner: prod-aisim-default-amd64-v1 + machine: x86_64 + scratch: true + - pool: arm64 + runner: prod-aisim-default-arm64-v1 + machine: aarch64 + scratch: true + runs-on: ${{ matrix.runner }} + timeout-minutes: 10 + container: + image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} + env: + HOME: /__w/aisim/aisim + USER: runner + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Verify source, job workspace, architecture and storage + shell: bash + env: + EXPECTED_MACHINE: ${{ matrix.machine }} + EXPECT_SCRATCH: ${{ matrix.scratch }} + run: | + set -euo pipefail + test "${GITHUB_REPOSITORY}" = ai-dynamo/aisim + test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" + test "$(pwd -P)" = /__w/aisim/aisim + test "${HOME}" = /__w/aisim/aisim + test -w "${HOME}" + test "$(uname -m)" = "${EXPECTED_MACHINE}" + probe=$(mktemp "${HOME}/.runner-canary.XXXXXX") + rm "${probe}" + if [[ "${EXPECT_SCRATCH}" == true ]]; then + test -w /scratch + probe=$(mktemp /scratch/runner-canary.XXXXXX) + rm "${probe}" + fi + printf 'Verified source %s on %s (%s) with writable HOME=%s\n' "${GITHUB_SHA}" "${RUNNER_NAME}" "${EXPECTED_MACHINE}" "${HOME}" + + shared: + if: github.repository == 'ai-dynamo/aisim' + name: Runner canary (shared) + runs-on: dynamo-cpu-4gi-amd64 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Verify shared runner admission, source and writable workspace + shell: bash + run: | + set -euo pipefail + test "${GITHUB_REPOSITORY}" = ai-dynamo/aisim + test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" + test "$(pwd -P)" = "$(realpath "${GITHUB_WORKSPACE}")" + test "$(uname -m)" = x86_64 + test -w "${HOME}" + probe=$(mktemp .runner-canary.XXXXXX) + rm "${probe}" + printf 'Verified source %s on shared runner %s\n' "${GITHUB_SHA}" "${RUNNER_NAME}" diff --git a/.github/workflows/simulation-performance.yml b/.github/workflows/simulation-performance.yml index 2942245948..37f734efda 100644 --- a/.github/workflows/simulation-performance.yml +++ b/.github/workflows/simulation-performance.yml @@ -165,7 +165,7 @@ jobs: needs: select if: needs.select.outputs.run_comparison == 'true' name: Build simulation wheel (${{ matrix.side }}) - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 30 strategy: fail-fast: false @@ -181,7 +181,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -260,7 +260,7 @@ jobs: needs: [select, build] if: needs.select.outputs.run_comparison == 'true' name: Simulation Performance (advisory) - runs-on: prod-aisimulate-default-amd64-v1 + runs-on: prod-aisim-default-amd64-v1 timeout-minutes: 75 defaults: run: @@ -268,7 +268,7 @@ jobs: container: image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} env: - HOME: /__w/aisimulate/aisimulate + HOME: /__w/aisim/aisim USER: runner env: BASE_SHA: ${{ needs.select.outputs.base_sha }}