ci: add engine version watch and nightly failure triage automation

.github/workflows/engine-version-watch.yml

@@ -0,0 +1,166 @@
+name: Engine Version Watch
+
+# Every 3 days: detect newer engine releases and have Claude file one
+# researched upgrade issue per engine (deduped against open issues, so a
+# version is only reported once).
+
+on:
+  schedule:
+    - cron: '23 8 */3 * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: engine-version-watch
+  cancel-in-progress: false
+
+jobs:
+  detect:
+    runs-on: k8s-runner-cpu
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    outputs:
+      pending: ${{ steps.dedup.outputs.pending }}
+      has_pending: ${{ steps.dedup.outputs.has_pending }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Install gh CLI
+        run: |
+          if ! command -v gh &>/dev/null; then
+            mkdir -p "$HOME/.local/bin"
+            GH_VERSION="2.74.0"
+            GH_SHA256="e55c9d49dc49c0b0fef0a9acd3510482fd9e27ff52ae80f8a6e838cd25b4cd89"
+            curl -fsSL --connect-timeout 10 --max-time 120 -o /tmp/gh.tgz \
+              "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz"
+            echo "${GH_SHA256}  /tmp/gh.tgz" | sha256sum --check --quiet
+            tar xzf /tmp/gh.tgz --strip-components=2 -C "$HOME/.local/bin" "gh_${GH_VERSION}_linux_amd64/bin/gh"
+            echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          fi
+
+      - name: Check versions
+        id: check
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          bash scripts/check_engine_versions.sh | tee versions.jsonl
+          jq -cs '[.[] | select(.update)]' versions.jsonl > updates.json
+          echo "updates=$(cat updates.json)" >> "$GITHUB_OUTPUT"
+
+      - name: Drop engines that already have an open issue
+        id: dedup
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh label create engine-watch --repo "$GITHUB_REPOSITORY" \
+            --description "Automated engine release tracking" --color 0e8a16 --force
+          gh label create enhancement --repo "$GITHUB_REPOSITORY" \
+            --description "New feature or request" --color a2eeef --force
+          # Exact substring match on fetched titles: GitHub's in:title search
+          # normalizes dots/hyphens, so "sglang 0.5.16" can match unrelated
+          # issues (or miss).
+          titles=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \
+            --label engine-watch --limit 200 --json title --jq '[.[].title]')
+          pending='[]'
+          while IFS= read -r row; do
+            engine=$(jq -r .engine <<<"$row")
+            latest=$(jq -r .latest <<<"$row")
+            existing=$(jq --arg t "$engine $latest" '[.[] | select(contains($t))] | length' <<<"$titles")
+            if [ "$existing" = "0" ]; then
+              pending=$(jq -c --argjson r "$row" '. + [$r]' <<<"$pending")
+            else
+              echo "Skipping $engine $latest: open issue exists"
+            fi
+          done < <(jq -c '.[]' updates.json)
+          echo "pending=$pending" >> "$GITHUB_OUTPUT"
+          echo "has_pending=$(jq 'length > 0' <<<"$pending")" >> "$GITHUB_OUTPUT"
+
+  research-and-file:
+    needs: detect
+    if: needs.detect.outputs.has_pending == 'true'
+    runs-on: k8s-runner-cpu
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      issues: write
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Export API key from pod env
+        run: |
+          if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
+            echo "::error::ANTHROPIC_API_KEY is not present in the runner pod environment"
+            exit 1
+          fi
+          echo "::add-mask::${ANTHROPIC_API_KEY}"
+          echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" >> "$GITHUB_ENV"
+
+      - name: Install gh CLI
+        run: |
+          if ! command -v gh &>/dev/null; then
+            mkdir -p "$HOME/.local/bin"
+            GH_VERSION="2.74.0"
+            GH_SHA256="e55c9d49dc49c0b0fef0a9acd3510482fd9e27ff52ae80f8a6e838cd25b4cd89"
+            curl -fsSL --connect-timeout 10 --max-time 120 -o /tmp/gh.tgz \
+              "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz"
+            echo "${GH_SHA256}  /tmp/gh.tgz" | sha256sum --check --quiet
+            tar xzf /tmp/gh.tgz --strip-components=2 -C "$HOME/.local/bin" "gh_${GH_VERSION}_linux_amd64/bin/gh"
+            echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          fi
+
+      - uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ env.ANTHROPIC_API_KEY }}
+          prompt: |
+            REPO: ${{ github.repository }}
+            PENDING ENGINE UPDATES (JSON): ${{ needs.detect.outputs.pending }}
+
+            For EACH entry, research the upgrade and create ONE GitHub issue.
+
+            Research per engine:
+            1. Locate every pin of the current version in this repo
+               (scripts/ci_install_<engine>.sh, .github/workflows/release-*-docker.yml,
+               grpc_servicer/pyproject.toml, docker/engine.Dockerfile).
+            2. Check whether nearby workaround comments are tied to the pinned
+               version (e.g. dependency band pins, forced reinstalls) and
+               whether the new release's dependency metadata makes them
+               obsolete — for PyPI packages fetch
+               https://pypi.org/pypi/<pkg>/<version>/json and inspect
+               requires_dist. For tensorrt-llm use https://pypi.nvidia.com.
+               For tokenspeed, `current`/`latest` are commit SHAs — summarize
+               `gh api repos/lightseekorg/tokenspeed/compare/<current>...<latest>`
+               (commit subjects only) and note that the torch pin in
+               ci_install_tokenspeed.sh must be re-checked against upstream.
+            3. Verify the release image/wheel actually exists (Docker Hub tag,
+               NGC wheel listing) before recommending it.
+
+            Then create the issue:
+              gh issue create --repo ${{ github.repository }} \
+                --label engine-watch --label enhancement \
+                --title "[engine-watch] <engine> <latest> available (pinned: <current>)" \
+                --body <researched body>
+            Body sections: current pin locations (file:line), what the new
+            release changes for us (workarounds now obsolete / new pins
+            required, with evidence), an upgrade checklist mirroring the
+            files above, and which e2e legs validate the bump. Keep it
+            factual and short; no speculation beyond the metadata checked.
+
+            Do NOT open pull requests. Do NOT edit repository files.
+            One issue per engine, nothing else.
+          claude_args: |
+            --model claude-opus-4-6
+            --max-turns 40
+            --allowedTools "Read,Glob,Grep,Bash,WebFetch,WebSearch,TaskCreate,TaskUpdate,TaskGet"
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/nightly-triage.yml

@@ -0,0 +1,161 @@
+name: Nightly Triage
+
+# Daily: collect recent nightly benchmark/eval failures and have Claude
+# triage them — classify each (regression vs infra/upstream flake vs
+# config), rerun clear one-off flakes once, and keep one rolling issue per
+# workflow so failures stop landing in a void. Runs after the ~7h bfcl/tau2
+# windows finish.
+
+on:
+  schedule:
+    - cron: '30 15 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: nightly-triage
+  cancel-in-progress: false
+
+env:
+  WATCHED_WORKFLOWS: nightly-bfcl.yml nightly-tau2.yml nightly-benchmark.yml nightly-docker.yml nightly-engine-docker.yml nightly-mlx-bench.yml
+
+jobs:
+  collect:
+    runs-on: k8s-runner-cpu
+    timeout-minutes: 10
+    permissions:
+      actions: read
+    outputs:
+      failures: ${{ steps.gather.outputs.failures }}
+      has_failures: ${{ steps.gather.outputs.has_failures }}
+    steps:
+      - name: Install gh CLI
+        run: |
+          if ! command -v gh &>/dev/null; then
+            mkdir -p "$HOME/.local/bin"
+            GH_VERSION="2.74.0"
+            GH_SHA256="e55c9d49dc49c0b0fef0a9acd3510482fd9e27ff52ae80f8a6e838cd25b4cd89"
+            curl -fsSL --connect-timeout 10 --max-time 120 -o /tmp/gh.tgz \
+              "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz"
+            echo "${GH_SHA256}  /tmp/gh.tgz" | sha256sum --check --quiet
+            tar xzf /tmp/gh.tgz --strip-components=2 -C "$HOME/.local/bin" "gh_${GH_VERSION}_linux_amd64/bin/gh"
+            echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          fi
+
+      - name: Gather failed nightly runs
+        id: gather
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # 48h lookback: a run still in progress at collection time (e.g.
+          # nightly-benchmark allows 24h) is picked up the next day; Claude
+          # dedups already-triaged run ids against the rolling issue.
+          since=$(date -u -d '48 hours ago' +%Y-%m-%dT%H:%M:%SZ)
+          failures='[]'
+          for wf in $WATCHED_WORKFLOWS; do
+            rows=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$wf" \
+              --created ">=$since" --limit 10 \
+              --json databaseId,conclusion,status,url,attempt,workflowName \
+              --jq '[.[] | select(.status == "completed" and (.conclusion | IN("success", "cancelled", "skipped") | not))
+                     | {workflow: .workflowName, run_id: .databaseId, attempt: .attempt, conclusion: .conclusion, url: .url}]')
+            failures=$(jq -c --argjson r "$rows" '. + $r' <<<"$failures")
+          done
+          echo "failures=$failures" >> "$GITHUB_OUTPUT"
+          echo "has_failures=$(jq 'length > 0' <<<"$failures")" >> "$GITHUB_OUTPUT"
+          jq . <<<"$failures"
+
+  triage:
+    needs: collect
+    if: needs.collect.outputs.has_failures == 'true'
+    runs-on: k8s-runner-cpu
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      actions: write
+      issues: write
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Export API key from pod env
+        run: |
+          if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
+            echo "::error::ANTHROPIC_API_KEY is not present in the runner pod environment"
+            exit 1
+          fi
+          echo "::add-mask::${ANTHROPIC_API_KEY}"
+          echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" >> "$GITHUB_ENV"
+
+      - name: Install gh CLI
+        run: |
+          if ! command -v gh &>/dev/null; then
+            mkdir -p "$HOME/.local/bin"
+            GH_VERSION="2.74.0"
+            GH_SHA256="e55c9d49dc49c0b0fef0a9acd3510482fd9e27ff52ae80f8a6e838cd25b4cd89"
+            curl -fsSL --connect-timeout 10 --max-time 120 -o /tmp/gh.tgz \
+              "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz"
+            echo "${GH_SHA256}  /tmp/gh.tgz" | sha256sum --check --quiet
+            tar xzf /tmp/gh.tgz --strip-components=2 -C "$HOME/.local/bin" "gh_${GH_VERSION}_linux_amd64/bin/gh"
+            echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          fi
+
+      - name: Ensure label
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh label create nightly-triage --repo "$GITHUB_REPOSITORY" \
+            --description "Automated nightly failure triage" --color d93f0b --force
+
+      - uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ env.ANTHROPIC_API_KEY }}
+          prompt: |
+            REPO: ${{ github.repository }}
+            FAILED NIGHTLY RUNS (last 48h, JSON): ${{ needs.collect.outputs.failures }}
+
+            Triage every run listed. For each:
+            1. Find the rolling issue first:
+               gh issue list --repo ${{ github.repository }} --state open
+                 --label nightly-triage --json number,title
+               and pick the exact title "[nightly-triage] <workflow name>".
+               If that issue already mentions this run_id in its body or
+               comments, the run was triaged on a previous day — skip it.
+            2. Fetch evidence:
+               `gh run view <run_id> --repo ${{ github.repository }} --log-failed | tail -300`
+               (fall back to `gh api .../jobs` for job names/conclusions if
+               logs are unavailable). Read the workflow file and the scripts
+               it calls when the failure points into them.
+            3. Classify with the log lines as evidence:
+               - infra/flake: runner lost, OOM-killed runner, network timeout,
+                 registry/HF download failure, upstream API 5xx
+               - regression: assertion/score threshold failures (bfcl/tau2
+                 score drops, benchmark deltas), build breaks in our code
+               - config/env: version conflicts, missing secrets, disk full
+            4. Rerun policy (conservative): if AND ONLY IF the failure is a
+               clear infra/flake and the run's attempt == 1, rerun it once:
+               `gh run rerun <run_id> --repo ${{ github.repository }} --failed`.
+               Never rerun regressions or second attempts.
+            5. Report per workflow in the rolling issue:
+               - If it exists, add a comment titled with today's date holding
+                 the day's triage (classification, key log lines, action
+                 taken, run links — always include the run_id). If the same
+                 failure signature already appears in recent comments, say
+                 so — recurring failures are the signal these issues exist
+                 to surface.
+               - Else create it:
+                 gh issue create --label nightly-triage
+                   --title "[nightly-triage] <workflow name>"
+                   --body <first day's triage>
+            Keep each day's entry short: verdict first, then evidence.
+            Do NOT edit repository files or open pull requests.
+          claude_args: |
+            --model claude-opus-4-6
+            --max-turns 60
+            --allowedTools "Read,Glob,Grep,Bash,WebFetch,WebSearch,TaskCreate,TaskUpdate,TaskGet"
+        env:
+          GH_TOKEN: ${{ github.token }}

scripts/check_engine_versions.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Compare pinned engine versions against the latest upstream releases.
+#
+# Emits one JSON object per line: {"engine","current","latest","update"}.
+# Pins are read from the same files the upgrade PRs edit, so a bump lands
+# here automatically. TokenSpeed is a source build pinned to a commit, so
+# its "versions" are SHAs and any divergence from upstream main counts as
+# an update.
+#
+# Requires: curl, jq; GH_TOKEN for the TokenSpeed upstream lookup.
+
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+CURL=(curl -fsS --connect-timeout 10 --max-time 60)
+
+require() { # name value
+    if [ -z "$2" ]; then
+        echo "ERROR: could not determine $1 (pin or upstream format changed?)" >&2
+        exit 1
+    fi
+}
+
+# Newest of two versions under PEP 440 pre-release ordering: rcN sorts
+# before its final release (GNU sort -V treats '~' as lowest).
+vmax() {
+    printf '%s\n%s\n' "$1" "$2" | sed 's/rc/~rc/' | sort -V | tail -1 | sed 's/~rc/rc/'
+}
+
+emit() { # engine current latest
+    local update=false
+    if [ "$1" = "tokenspeed" ]; then
+        [ "$2" != "$3" ] && update=true
+    else
+        [ "$(vmax "$2" "$3")" != "$2" ] && update=true
+    fi
+    jq -cn --arg e "$1" --arg c "$2" --arg l "$3" --argjson u "$update" \
+        '{engine: $e, current: $c, latest: $l, update: $u}'
+}
+
+sglang_current=$(sed -n 's/.*"sglang\[all\]==\([^"]*\)".*/\1/p' scripts/ci_install_sglang.sh | head -1)
+require "sglang pin" "$sglang_current"
+sglang_latest=$("${CURL[@]}" https://pypi.org/pypi/sglang/json | jq -r .info.version)
+require "sglang latest" "$sglang_latest"
+emit sglang "$sglang_current" "$sglang_latest"
+
+vllm_current=$(sed -n "s/.*default: 'vllm\/vllm-openai:v\([0-9.]*\)'.*/\1/p" .github/workflows/release-vllm-docker.yml | head -1)
+require "vllm pin" "$vllm_current"
+vllm_latest=$("${CURL[@]}" https://pypi.org/pypi/vllm/json | jq -r .info.version)
+require "vllm latest" "$vllm_latest"
+emit vllm "$vllm_current" "$vllm_latest"
+
+trtllm_current=$(sed -n 's/^TRTLLM_VERSION="\(.*\)"$/\1/p' scripts/ci_install_trtllm.sh | head -1)
+require "tensorrt-llm pin" "$trtllm_current"
+trtllm_latest=$("${CURL[@]}" https://pypi.nvidia.com/tensorrt-llm/ \
+    | grep -o 'tensorrt_llm-[0-9][^-]*' | sed 's/tensorrt_llm-//;s/rc/~rc/' | sort -uV | tail -1 | sed 's/~rc/rc/')
+require "tensorrt-llm latest" "$trtllm_latest"
+emit tensorrt-llm "$trtllm_current" "$trtllm_latest"
+
+tokenspeed_current=$(sed -n 's/.*TOKENSPEED_REF:-\([0-9a-f]*\)}.*/\1/p' scripts/ci_install_tokenspeed.sh | head -1)
+require "tokenspeed pin" "$tokenspeed_current"
+tokenspeed_latest=$(gh api repos/lightseekorg/tokenspeed/commits/main --jq .sha)
+require "tokenspeed latest" "$tokenspeed_latest"
+emit tokenspeed "$tokenspeed_current" "$tokenspeed_latest"