chore: AUT-2539 nightly sync main into dev (2026-10-13)

.github/copy-pr-bot.yaml

@@ -1,4 +1,4 @@
 enabled: true
 auto_sync_draft: false
 auto_sync_ready: true
-trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "Connor-XY", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JF-D", "JRD971000", "Leili", "Mellonta", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "WanZzzzzz", "Wohox", "YangFei1990", "ZhiyuLi-Nvidia", "adistomar", "ahmadki", "aklife97", "alokpathy", "ananthsub", "anlthms", "aroshanghias-nvd", "ashehper", "asolergi-nv", "athitten", "balasaajay", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "desh2608", "dimapihtar", "dingqingy-nv", "duncanriach", "ehosseiniasl", "erhoo82", "ericharper", "fanshiqing", "faradawn", "fitsumreda", "freewym", "frsun-nvda", "gautham-kollu", "gdengk", "goelarushi", "guihong-nv", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jenchen13", "jiaji-huang", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kajalj22", "kamran-nvidia", "kevalmorabia97", "kevjshih", "kingformatty", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwyss-nvidia", "lauradang", "layalir", "lhb8125", "liding-nv", "lmcafee-nvidia", "maanug-nv", "macandro96", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "minitu", "mkhona-nvidia", "nanz-nv", "ntajbakhsh", "nvcsathe", "parthmannan", "philipcmonk", "prajwal1210", "pthombre", "rapatel", "rhewett-nv", "rogerwaleffe", "sajadn", "sanandaraj5597", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "sharathts", "sheliang-nv", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sraman-rgb", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "tomlifu", "trintamaki", "tylerpoon", "wdykas", "wplf", "wujingyue", "xiaoyao0115", "xuantengh", "xuwchen", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yqwangustc", "yueshen2016", "yuzhongw-nvidia", "zhehuaichen", "zhongbozhu", "DanialTaheri", "niyunsheng", "svcnvidia-nemo-ci", "vasunvidia"]
+trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "Connor-XY", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JF-D", "JRD971000", "Leili", "Mellonta", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "WanZzzzzz", "Wohox", "YangFei1990", "ZhiyuLi-Nvidia", "adistomar", "ahmadki", "aklife97", "alokpathy", "ananthsub", "anlthms", "aroshanghias-nvd", "ashehper", "asolergi-nv", "athitten", "balasaajay", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "desh2608", "dimapihtar", "dingqingy-nv", "duncanriach", "ehosseiniasl", "erhoo82", "ericharper", "fanshiqing", "faradawn", "fitsumreda", "freewym", "frsun-nvda", "gautham-kollu", "gdengk", "goelarushi", "guihong-nv", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jenchen13", "jiaji-huang", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kajalj22", "kamran-nvidia", "kevalmorabia97", "kevjshih", "kingformatty", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwangjunahn", "kwyss-nvidia", "lauradang", "layalir", "lhb8125", "liding-nv", "lmcafee-nvidia", "maanug-nv", "macandro96", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "minitu", "mkhona-nvidia", "nanz-nv", "ntajbakhsh", "nvcsathe", "parthmannan", "philipcmonk", "prajwal1210", "pthombre", "rapatel", "rhewett-nv", "rogerwaleffe", "sajadn", "sanandaraj5597", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "sharathts", "sheliang-nv", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sraman-rgb", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "tomlifu", "trintamaki", "tylerpoon", "wdykas", "wplf", "wujingyue", "xiaoyao0115", "xuantengh", "xuwchen", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yqwangustc", "yueshen2016", "yuzhongw-nvidia", "zhehuaichen", "zhongbozhu", "DanialTaheri", "niyunsheng", "svcnvidia-nemo-ci", "vasunvidia"]

.github/oncall_schedule.json

@@ -1,12 +1,4 @@
 [
-    {
-        "user": "janEbert",
-        "date": "2026-08-05"
-    },
-    {
-        "user": "maanug-nv",
-        "date": "2026-08-12"
-    },
     {
         "user": "Phlip79",
         "date": "2026-08-19"
@@ -46,5 +38,13 @@
     {
         "user": "janEbert",
         "date": "2026-10-21"
+    },
+    {
+        "user": "maanug-nv",
+        "date": "2026-10-28"
+    },
+    {
+        "user": "Phlip79",
+        "date": "2026-11-04"
     }
 ]

.github/scripts/cancel_superseded_merge_group_runs.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+"""Cancel active workflow runs for superseded generations of one queued PR."""
+
+import json
+import os
+import re
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+_ACTIVE_STATUSES = ("requested", "queued", "in_progress", "waiting", "pending")
+_MERGE_GROUP_REF = re.compile(r"^gh-readonly-queue/.+/pr-(\d+)-[0-9a-f]+$", re.IGNORECASE)
+
+
+class GitHubApiError(RuntimeError):
+    """Report a failed GitHub REST API request."""
+
+    def __init__(self, status: int, message: str) -> None:
+        super().__init__(message)
+        self.status = status
+
+
+class GitHubClient:
+    """Minimal GitHub Actions REST API client."""
+
+    def __init__(self, api_url: str, repository: str, token: str) -> None:
+        owner, name = repository.split("/", maxsplit=1)
+        self._repository_path = f"repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}"
+        self._api_url = api_url.rstrip("/")
+        self._headers = {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+
+    def _request(self, path: str, method: str = "GET") -> dict[str, object] | None:
+        for attempt in range(5):
+            request = urllib.request.Request(
+                f"{self._api_url}/{self._repository_path}/{path}",
+                data=b"" if method == "POST" else None,
+                headers=self._headers,
+                method=method,
+            )
+            try:
+                with urllib.request.urlopen(request, timeout=30) as response:
+                    payload = response.read()
+                return json.loads(payload) if payload else None
+            except urllib.error.HTTPError as error:
+                details = error.read().decode("utf-8", errors="replace")
+                if error.code != 429 and error.code < 500:
+                    raise GitHubApiError(
+                        error.code, f"GitHub API {method} {path} failed: {details}"
+                    ) from error
+                if attempt == 4:
+                    raise GitHubApiError(
+                        error.code, f"GitHub API {method} {path} failed after retries: {details}"
+                    ) from error
+                delay = min(int(error.headers.get("Retry-After", 2**attempt)), 30)
+            except (TimeoutError, urllib.error.URLError) as error:
+                if attempt == 4:
+                    raise RuntimeError(f"GitHub API {method} {path} failed after retries: {error}") from error
+                delay = 2**attempt
+
+            print(f"GitHub API {method} {path} failed transiently; retrying in {delay}s")
+            time.sleep(delay)
+
+        raise AssertionError("unreachable")
+
+    def _list_runs(self, endpoint: str, parameters: dict[str, str]) -> list[dict[str, object]]:
+        runs: list[dict[str, object]] = []
+        page = 1
+        while True:
+            query = urllib.parse.urlencode({**parameters, "per_page": "100", "page": str(page)})
+            response = self._request(f"{endpoint}?{query}")
+            if not isinstance(response, dict) or not isinstance(response.get("workflow_runs"), list):
+                raise RuntimeError(f"GitHub API returned an invalid workflow-runs response for {endpoint}")
+
+            page_runs = response["workflow_runs"]
+            runs.extend(page_runs)
+            if len(page_runs) < 100:
+                return runs
+            page += 1
+
+    def list_trigger_workflow_runs(
+        self, workflow_id: int, head_branches: set[str]
+    ) -> list[dict[str, object]]:
+        """List merge-group runs that establish the ordering of active generations."""
+
+        runs: list[dict[str, object]] = []
+        for head_branch in sorted(head_branches):
+            runs.extend(
+                self._list_runs(
+                    f"actions/workflows/{workflow_id}/runs",
+                    {"branch": head_branch, "event": "merge_group"},
+                )
+            )
+        return runs
+
+    def list_active_merge_group_runs(self) -> list[dict[str, object]]:
+        """List all non-terminal merge-group workflow runs in the repository."""
+
+        runs_by_id: dict[int, dict[str, object]] = {}
+        for status in _ACTIVE_STATUSES:
+            for run in self._list_runs("actions/runs", {"event": "merge_group", "status": status}):
+                runs_by_id[int(run["id"])] = run
+        return list(runs_by_id.values())
+
+    def cancel_run(self, run_id: int) -> None:
+        """Request cancellation of one workflow run."""
+
+        self._request(f"actions/runs/{run_id}/cancel", method="POST")
+
+
+def _pr_number(head_branch: object) -> int | None:
+    match = _MERGE_GROUP_REF.fullmatch(str(head_branch))
+    return int(match.group(1)) if match else None
+
+
+def _run_order(run: dict[str, object]) -> tuple[str, int]:
+    return str(run["created_at"]), int(run["id"])
+
+
+def _select_superseded_runs(
+    active_runs: list[dict[str, object]],
+    trigger_workflow_runs: list[dict[str, object]],
+    current_run: dict[str, object],
+) -> list[dict[str, object]]:
+    current_head = str(current_run["head_branch"])
+    current_pr = _pr_number(current_head)
+    if current_pr is None:
+        raise ValueError(f"Unexpected merge-group head branch: {current_head}")
+
+    generation_order: dict[str, tuple[str, int]] = {}
+    for run in trigger_workflow_runs:
+        head_branch = str(run.get("head_branch", ""))
+        if _pr_number(head_branch) != current_pr:
+            continue
+        run_order = _run_order(run)
+        previous_order = generation_order.get(head_branch)
+        if previous_order is None or run_order < previous_order:
+            generation_order[head_branch] = run_order
+
+    current_order = _run_order(current_run)
+    generation_order[current_head] = min(generation_order.get(current_head, current_order), current_order)
+    if any(head != current_head and order > current_order for head, order in generation_order.items()):
+        print(f"A newer merge-group generation exists for PR #{current_pr}; this cleanup is stale")
+        return []
+
+    superseded = []
+    for run in active_runs:
+        head_branch = str(run.get("head_branch", ""))
+        if head_branch == current_head or _pr_number(head_branch) != current_pr:
+            continue
+
+        old_generation_order = generation_order.get(head_branch)
+        if old_generation_order is None:
+            print(f"Skipping run {run.get('id')} because its merge-group generation was not found")
+            continue
+        if old_generation_order < current_order:
+            superseded.append(run)
+
+    return sorted(superseded, key=_run_order)
+
+
+def main() -> None:
+    """Cancel active runs belonging to older merge-group generations of the triggering PR."""
+
+    current_run = {
+        "id": int(os.environ["TRIGGER_RUN_ID"]),
+        "head_branch": os.environ["TRIGGER_HEAD_BRANCH"],
+        "created_at": os.environ["TRIGGER_CREATED_AT"],
+    }
+    client = GitHubClient(
+        api_url=os.environ.get("GITHUB_API_URL", "https://api.github.com"),
+        repository=os.environ["GITHUB_REPOSITORY"],
+        token=os.environ["GITHUB_TOKEN"],
+    )
+
+    active_runs = client.list_active_merge_group_runs()
+    current_pr = _pr_number(current_run["head_branch"])
+    if current_pr is None:
+        raise ValueError(f"Unexpected merge-group head branch: {current_run['head_branch']}")
+    active_heads = {
+        str(run.get("head_branch", "")) for run in active_runs if _pr_number(run.get("head_branch")) == current_pr
+    }
+    active_heads.add(str(current_run["head_branch"]))
+    trigger_runs = client.list_trigger_workflow_runs(int(os.environ["TRIGGER_WORKFLOW_ID"]), active_heads)
+    superseded_runs = _select_superseded_runs(active_runs, trigger_runs, current_run)
+
+    canceled = 0
+    for run in superseded_runs:
+        run_id = int(run["id"])
+        try:
+            client.cancel_run(run_id)
+        except GitHubApiError as error:
+            if error.status == 409:
+                print(f"Run {run_id} became terminal before cancellation")
+                continue
+            raise
+        print(f"Canceled superseded {run.get('name', 'workflow')} run {run_id} ({run.get('head_branch')})")
+        canceled += 1
+
+    print(f"Canceled {canceled} superseded merge-group run(s)")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/test_approve_test_queue.sh

@@ -3,6 +3,13 @@ set -euo pipefail
 
 readonly WORKFLOW=${WORKFLOW:-.github/workflows/cicd-approve-test-queue.yml}
 readonly EXTERNAL_QUEUE='          - branch: all'
+readonly WORKFLOW_RUN_EVENT='  workflow_run:'
+readonly CICD_COMPLETION_TRIGGER='    workflows: ["CICD Megatron-LM"]'
+readonly COMPLETED_ACTIVITY='    types: [completed]'
+readonly CONCURRENCY_GROUP='  group: approve-test-queue'
+readonly SERIALIZED_RUNS='  cancel-in-progress: false'
+readonly INTERNAL_SERVICE_ACCOUNT='          INTERNAL_SERVICE_ACCOUNTS = {"svcnemo-autobot"}'
+readonly INTERNAL_SERVICE_ACCOUNT_CHECK='              return login in INTERNAL_SERVICE_ACCOUNTS or any('
 
 if [[ $(/usr/bin/grep -c -F "$EXTERNAL_QUEUE" "$WORKFLOW") -ne 1 ]]; then
   echo "Approve Test Queue must define exactly one global external worker" >&2
@@ -19,4 +26,26 @@ if [[ $(/usr/bin/grep -c -F 'if CONTRIBUTOR_TYPE == "external":' "$WORKFLOW") -n
   exit 1
 fi
 
-echo "Approve Test Queue uses one global external worker"
+for expected in \
+  "$WORKFLOW_RUN_EVENT" \
+  "$CICD_COMPLETION_TRIGGER" \
+  "$COMPLETED_ACTIVITY" \
+  "$CONCURRENCY_GROUP" \
+  "$SERIALIZED_RUNS"; do
+  if [[ $(/usr/bin/grep -c -F "$expected" "$WORKFLOW") -ne 1 ]]; then
+    echo "Approve Test Queue must wake on completed CICD runs without racing approvals: $expected" >&2
+    exit 1
+  fi
+done
+
+if [[ $(/usr/bin/grep -c -F "$INTERNAL_SERVICE_ACCOUNT" "$WORKFLOW") -ne 1 ]]; then
+  echo "Approve Test Queue must classify svcnemo-autobot as internal without relying on the SSO export" >&2
+  exit 1
+fi
+
+if [[ $(/usr/bin/grep -c -F "$INTERNAL_SERVICE_ACCOUNT_CHECK" "$WORKFLOW") -ne 1 ]]; then
+  echo "Approve Test Queue must apply its internal service-account allowlist" >&2
+  exit 1
+fi
+
+echo "Approve Test Queue uses one global external worker, recognizes its service account, and wakes on completion"

.github/scripts/test_cancel_superseded_merge_group_runs.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+import unittest
+from pathlib import Path
+
+from cancel_superseded_merge_group_runs import _pr_number, _select_superseded_runs
+
+
+def _run(run_id: int, head_branch: str, created_at: str, name: str = "CICD Megatron-LM") -> dict[str, object]:
+    return {
+        "id": run_id,
+        "event": "merge_group",
+        "head_branch": head_branch,
+        "created_at": created_at,
+        "name": name,
+    }
+
+
+class TestCancelSupersededMergeGroupRuns(unittest.TestCase):
+    def setUp(self) -> None:
+        self.old_head = "gh-readonly-queue/main/pr-123-a1d1234"
+        self.current_head = "gh-readonly-queue/main/pr-123-b2e5678"
+        self.current_run = _run(200, self.current_head, "2026-08-13T22:00:00Z")
+        self.trigger_runs = [
+            self.current_run,
+            _run(100, self.old_head, "2026-08-13T21:00:00Z"),
+        ]
+
+    def test_extracts_pr_number_only_from_merge_group_refs(self) -> None:
+        self.assertEqual(_pr_number(self.current_head), 123)
+        self.assertIsNone(_pr_number("pull-request/123"))
+        self.assertIsNone(_pr_number("feature/pr-123-not-a-merge-group"))
+
+    def test_selects_all_active_runs_from_the_older_generation(self) -> None:
+        old_cicd = _run(100, self.old_head, "2026-08-13T21:00:00Z")
+        old_release = _run(250, self.old_head, "2026-08-13T22:01:00Z", name="Release")
+        current_release = _run(201, self.current_head, "2026-08-13T22:00:01Z", name="Release")
+        other_pr = _run(50, "gh-readonly-queue/main/pr-456-a7e1234", "2026-08-13T20:00:00Z")
+
+        selected = _select_superseded_runs(
+            [old_cicd, old_release, self.current_run, current_release, other_pr],
+            self.trigger_runs,
+            self.current_run,
+        )
+
+        self.assertEqual([run["id"] for run in selected], [100, 250])
+
+    def test_stale_cleanup_does_not_cancel_a_newer_generation(self) -> None:
+        newer_head = "gh-readonly-queue/main/pr-123-c3f9abc"
+        newer_run = _run(300, newer_head, "2026-08-13T23:00:00Z")
+
+        selected = _select_superseded_runs(
+            [self.current_run, newer_run],
+            [newer_run, *self.trigger_runs],
+            self.current_run,
+        )
+
+        self.assertEqual(selected, [])
+
+    def test_rerun_does_not_change_the_generation_order(self) -> None:
+        old_rerun = _run(400, self.old_head, "2026-08-13T23:00:00Z")
+
+        selected = _select_superseded_runs(
+            [old_rerun],
+            [old_rerun, *self.trigger_runs],
+            self.current_run,
+        )
+
+        self.assertEqual([run["id"] for run in selected], [400])
+
+    def test_skips_an_active_generation_without_trusted_ordering_data(self) -> None:
+        unknown_head = "gh-readonly-queue/main/pr-123-d4a1111"
+        unknown_run = _run(50, unknown_head, "2026-08-13T20:00:00Z")
+
+        selected = _select_superseded_runs([unknown_run], self.trigger_runs, self.current_run)
+
+        self.assertEqual(selected, [])
+
+    def test_workflow_runs_from_the_default_branch_with_minimal_write_permission(self) -> None:
+        workflow = Path(".github/workflows/cicd-cancel-superseded-merge-groups.yml").read_text()
+
+        self.assertIn('workflows: ["CICD Megatron-LM"]', workflow)
+        self.assertIn("types: [requested]", workflow)
+        self.assertIn("github.event.workflow_run.event == 'merge_group'", workflow)
+        self.assertIn("actions: write", workflow)
+        self.assertIn("ref: ${{ github.event.repository.default_branch }}", workflow)
+        self.assertIn("persist-credentials: false", workflow)
+
+
+if __name__ == "__main__":
+    unittest.main()