diff --git a/.github/workflows/engine-version-watch.yml b/.github/workflows/engine-version-watch.yml new file mode 100644 index 0000000000..47d5f2e7a9 --- /dev/null +++ b/.github/workflows/engine-version-watch.yml @@ -0,0 +1,166 @@ +name: Engine Version Watch + +# Every 3 days: detect newer engine releases and have Claude file one +# researched upgrade issue per engine (deduped against open issues, so a +# version is only reported once). + +on: + schedule: + - cron: '23 8 */3 * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: engine-version-watch + cancel-in-progress: false + +jobs: + detect: + runs-on: k8s-runner-cpu + timeout-minutes: 10 + permissions: + contents: read + issues: write + outputs: + pending: ${{ steps.dedup.outputs.pending }} + has_pending: ${{ steps.dedup.outputs.has_pending }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Install gh CLI + run: | + if ! command -v gh &>/dev/null; then + mkdir -p "$HOME/.local/bin" + GH_VERSION="2.74.0" + GH_SHA256="e55c9d49dc49c0b0fef0a9acd3510482fd9e27ff52ae80f8a6e838cd25b4cd89" + curl -fsSL --connect-timeout 10 --max-time 120 -o /tmp/gh.tgz \ + "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" + echo "${GH_SHA256} /tmp/gh.tgz" | sha256sum --check --quiet + tar xzf /tmp/gh.tgz --strip-components=2 -C "$HOME/.local/bin" "gh_${GH_VERSION}_linux_amd64/bin/gh" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + fi + + - name: Check versions + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + bash scripts/check_engine_versions.sh | tee versions.jsonl + jq -cs '[.[] | select(.update)]' versions.jsonl > updates.json + echo "updates=$(cat updates.json)" >> "$GITHUB_OUTPUT" + + - name: Drop engines that already have an open issue + id: dedup + env: + GH_TOKEN: ${{ github.token }} + run: | + gh label create engine-watch --repo "$GITHUB_REPOSITORY" \ + --description "Automated engine release tracking" --color 0e8a16 --force + gh label create enhancement --repo "$GITHUB_REPOSITORY" \ + --description "New feature or request" --color a2eeef --force + # Exact substring match on fetched titles: GitHub's in:title search + # normalizes dots/hyphens, so "sglang 0.5.16" can match unrelated + # issues (or miss). + titles=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --label engine-watch --limit 200 --json title --jq '[.[].title]') + pending='[]' + while IFS= read -r row; do + engine=$(jq -r .engine <<<"$row") + latest=$(jq -r .latest <<<"$row") + existing=$(jq --arg t "$engine $latest" '[.[] | select(contains($t))] | length' <<<"$titles") + if [ "$existing" = "0" ]; then + pending=$(jq -c --argjson r "$row" '. + [$r]' <<<"$pending") + else + echo "Skipping $engine $latest: open issue exists" + fi + done < <(jq -c '.[]' updates.json) + echo "pending=$pending" >> "$GITHUB_OUTPUT" + echo "has_pending=$(jq 'length > 0' <<<"$pending")" >> "$GITHUB_OUTPUT" + + research-and-file: + needs: detect + if: needs.detect.outputs.has_pending == 'true' + runs-on: k8s-runner-cpu + timeout-minutes: 30 + permissions: + contents: read + issues: write + id-token: write + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Export API key from pod env + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "::error::ANTHROPIC_API_KEY is not present in the runner pod environment" + exit 1 + fi + echo "::add-mask::${ANTHROPIC_API_KEY}" + echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" >> "$GITHUB_ENV" + + - name: Install gh CLI + run: | + if ! command -v gh &>/dev/null; then + mkdir -p "$HOME/.local/bin" + GH_VERSION="2.74.0" + GH_SHA256="e55c9d49dc49c0b0fef0a9acd3510482fd9e27ff52ae80f8a6e838cd25b4cd89" + curl -fsSL --connect-timeout 10 --max-time 120 -o /tmp/gh.tgz \ + "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" + echo "${GH_SHA256} /tmp/gh.tgz" | sha256sum --check --quiet + tar xzf /tmp/gh.tgz --strip-components=2 -C "$HOME/.local/bin" "gh_${GH_VERSION}_linux_amd64/bin/gh" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + fi + + - uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ env.ANTHROPIC_API_KEY }} + prompt: | + REPO: ${{ github.repository }} + PENDING ENGINE UPDATES (JSON): ${{ needs.detect.outputs.pending }} + + For EACH entry, research the upgrade and create ONE GitHub issue. + + Research per engine: + 1. Locate every pin of the current version in this repo + (scripts/ci_install_.sh, .github/workflows/release-*-docker.yml, + grpc_servicer/pyproject.toml, docker/engine.Dockerfile). + 2. Check whether nearby workaround comments are tied to the pinned + version (e.g. dependency band pins, forced reinstalls) and + whether the new release's dependency metadata makes them + obsolete — for PyPI packages fetch + https://pypi.org/pypi///json and inspect + requires_dist. For tensorrt-llm use https://pypi.nvidia.com. + For tokenspeed, `current`/`latest` are commit SHAs — summarize + `gh api repos/lightseekorg/tokenspeed/compare/...` + (commit subjects only) and note that the torch pin in + ci_install_tokenspeed.sh must be re-checked against upstream. + 3. Verify the release image/wheel actually exists (Docker Hub tag, + NGC wheel listing) before recommending it. + + Then create the issue: + gh issue create --repo ${{ github.repository }} \ + --label engine-watch --label enhancement \ + --title "[engine-watch] available (pinned: )" \ + --body + Body sections: current pin locations (file:line), what the new + release changes for us (workarounds now obsolete / new pins + required, with evidence), an upgrade checklist mirroring the + files above, and which e2e legs validate the bump. Keep it + factual and short; no speculation beyond the metadata checked. + + Do NOT open pull requests. Do NOT edit repository files. + One issue per engine, nothing else. + claude_args: | + --model claude-opus-4-6 + --max-turns 40 + --allowedTools "Read,Glob,Grep,Bash,WebFetch,WebSearch,TaskCreate,TaskUpdate,TaskGet" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/nightly-triage.yml b/.github/workflows/nightly-triage.yml new file mode 100644 index 0000000000..bcd0b0bff5 --- /dev/null +++ b/.github/workflows/nightly-triage.yml @@ -0,0 +1,161 @@ +name: Nightly Triage + +# Daily: collect recent nightly benchmark/eval failures and have Claude +# triage them — classify each (regression vs infra/upstream flake vs +# config), rerun clear one-off flakes once, and keep one rolling issue per +# workflow so failures stop landing in a void. Runs after the ~7h bfcl/tau2 +# windows finish. + +on: + schedule: + - cron: '30 15 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-triage + cancel-in-progress: false + +env: + WATCHED_WORKFLOWS: nightly-bfcl.yml nightly-tau2.yml nightly-benchmark.yml nightly-docker.yml nightly-engine-docker.yml nightly-mlx-bench.yml + +jobs: + collect: + runs-on: k8s-runner-cpu + timeout-minutes: 10 + permissions: + actions: read + outputs: + failures: ${{ steps.gather.outputs.failures }} + has_failures: ${{ steps.gather.outputs.has_failures }} + steps: + - name: Install gh CLI + run: | + if ! command -v gh &>/dev/null; then + mkdir -p "$HOME/.local/bin" + GH_VERSION="2.74.0" + GH_SHA256="e55c9d49dc49c0b0fef0a9acd3510482fd9e27ff52ae80f8a6e838cd25b4cd89" + curl -fsSL --connect-timeout 10 --max-time 120 -o /tmp/gh.tgz \ + "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" + echo "${GH_SHA256} /tmp/gh.tgz" | sha256sum --check --quiet + tar xzf /tmp/gh.tgz --strip-components=2 -C "$HOME/.local/bin" "gh_${GH_VERSION}_linux_amd64/bin/gh" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + fi + + - name: Gather failed nightly runs + id: gather + env: + GH_TOKEN: ${{ github.token }} + run: | + # 48h lookback: a run still in progress at collection time (e.g. + # nightly-benchmark allows 24h) is picked up the next day; Claude + # dedups already-triaged run ids against the rolling issue. + since=$(date -u -d '48 hours ago' +%Y-%m-%dT%H:%M:%SZ) + failures='[]' + for wf in $WATCHED_WORKFLOWS; do + rows=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$wf" \ + --created ">=$since" --limit 10 \ + --json databaseId,conclusion,status,url,attempt,workflowName \ + --jq '[.[] | select(.status == "completed" and (.conclusion | IN("success", "cancelled", "skipped") | not)) + | {workflow: .workflowName, run_id: .databaseId, attempt: .attempt, conclusion: .conclusion, url: .url}]') + failures=$(jq -c --argjson r "$rows" '. + $r' <<<"$failures") + done + echo "failures=$failures" >> "$GITHUB_OUTPUT" + echo "has_failures=$(jq 'length > 0' <<<"$failures")" >> "$GITHUB_OUTPUT" + jq . <<<"$failures" + + triage: + needs: collect + if: needs.collect.outputs.has_failures == 'true' + runs-on: k8s-runner-cpu + timeout-minutes: 45 + permissions: + contents: read + actions: write + issues: write + id-token: write + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Export API key from pod env + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "::error::ANTHROPIC_API_KEY is not present in the runner pod environment" + exit 1 + fi + echo "::add-mask::${ANTHROPIC_API_KEY}" + echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" >> "$GITHUB_ENV" + + - name: Install gh CLI + run: | + if ! command -v gh &>/dev/null; then + mkdir -p "$HOME/.local/bin" + GH_VERSION="2.74.0" + GH_SHA256="e55c9d49dc49c0b0fef0a9acd3510482fd9e27ff52ae80f8a6e838cd25b4cd89" + curl -fsSL --connect-timeout 10 --max-time 120 -o /tmp/gh.tgz \ + "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" + echo "${GH_SHA256} /tmp/gh.tgz" | sha256sum --check --quiet + tar xzf /tmp/gh.tgz --strip-components=2 -C "$HOME/.local/bin" "gh_${GH_VERSION}_linux_amd64/bin/gh" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + fi + + - name: Ensure label + env: + GH_TOKEN: ${{ github.token }} + run: | + gh label create nightly-triage --repo "$GITHUB_REPOSITORY" \ + --description "Automated nightly failure triage" --color d93f0b --force + + - uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ env.ANTHROPIC_API_KEY }} + prompt: | + REPO: ${{ github.repository }} + FAILED NIGHTLY RUNS (last 48h, JSON): ${{ needs.collect.outputs.failures }} + + Triage every run listed. For each: + 1. Find the rolling issue first: + gh issue list --repo ${{ github.repository }} --state open + --label nightly-triage --json number,title + and pick the exact title "[nightly-triage] ". + If that issue already mentions this run_id in its body or + comments, the run was triaged on a previous day — skip it. + 2. Fetch evidence: + `gh run view --repo ${{ github.repository }} --log-failed | tail -300` + (fall back to `gh api .../jobs` for job names/conclusions if + logs are unavailable). Read the workflow file and the scripts + it calls when the failure points into them. + 3. Classify with the log lines as evidence: + - infra/flake: runner lost, OOM-killed runner, network timeout, + registry/HF download failure, upstream API 5xx + - regression: assertion/score threshold failures (bfcl/tau2 + score drops, benchmark deltas), build breaks in our code + - config/env: version conflicts, missing secrets, disk full + 4. Rerun policy (conservative): if AND ONLY IF the failure is a + clear infra/flake and the run's attempt == 1, rerun it once: + `gh run rerun --repo ${{ github.repository }} --failed`. + Never rerun regressions or second attempts. + 5. Report per workflow in the rolling issue: + - If it exists, add a comment titled with today's date holding + the day's triage (classification, key log lines, action + taken, run links — always include the run_id). If the same + failure signature already appears in recent comments, say + so — recurring failures are the signal these issues exist + to surface. + - Else create it: + gh issue create --label nightly-triage + --title "[nightly-triage] " + --body + Keep each day's entry short: verdict first, then evidence. + Do NOT edit repository files or open pull requests. + claude_args: | + --model claude-opus-4-6 + --max-turns 60 + --allowedTools "Read,Glob,Grep,Bash,WebFetch,WebSearch,TaskCreate,TaskUpdate,TaskGet" + env: + GH_TOKEN: ${{ github.token }} diff --git a/scripts/check_engine_versions.sh b/scripts/check_engine_versions.sh new file mode 100755 index 0000000000..e966738a08 --- /dev/null +++ b/scripts/check_engine_versions.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Compare pinned engine versions against the latest upstream releases. +# +# Emits one JSON object per line: {"engine","current","latest","update"}. +# Pins are read from the same files the upgrade PRs edit, so a bump lands +# here automatically. TokenSpeed is a source build pinned to a commit, so +# its "versions" are SHAs and any divergence from upstream main counts as +# an update. +# +# Requires: curl, jq; GH_TOKEN for the TokenSpeed upstream lookup. + +set -euo pipefail +cd "$(dirname "$0")/.." + +CURL=(curl -fsS --connect-timeout 10 --max-time 60) + +require() { # name value + if [ -z "$2" ]; then + echo "ERROR: could not determine $1 (pin or upstream format changed?)" >&2 + exit 1 + fi +} + +# Newest of two versions under PEP 440 pre-release ordering: rcN sorts +# before its final release (GNU sort -V treats '~' as lowest). +vmax() { + printf '%s\n%s\n' "$1" "$2" | sed 's/rc/~rc/' | sort -V | tail -1 | sed 's/~rc/rc/' +} + +emit() { # engine current latest + local update=false + if [ "$1" = "tokenspeed" ]; then + [ "$2" != "$3" ] && update=true + else + [ "$(vmax "$2" "$3")" != "$2" ] && update=true + fi + jq -cn --arg e "$1" --arg c "$2" --arg l "$3" --argjson u "$update" \ + '{engine: $e, current: $c, latest: $l, update: $u}' +} + +sglang_current=$(sed -n 's/.*"sglang\[all\]==\([^"]*\)".*/\1/p' scripts/ci_install_sglang.sh | head -1) +require "sglang pin" "$sglang_current" +sglang_latest=$("${CURL[@]}" https://pypi.org/pypi/sglang/json | jq -r .info.version) +require "sglang latest" "$sglang_latest" +emit sglang "$sglang_current" "$sglang_latest" + +vllm_current=$(sed -n "s/.*default: 'vllm\/vllm-openai:v\([0-9.]*\)'.*/\1/p" .github/workflows/release-vllm-docker.yml | head -1) +require "vllm pin" "$vllm_current" +vllm_latest=$("${CURL[@]}" https://pypi.org/pypi/vllm/json | jq -r .info.version) +require "vllm latest" "$vllm_latest" +emit vllm "$vllm_current" "$vllm_latest" + +trtllm_current=$(sed -n 's/^TRTLLM_VERSION="\(.*\)"$/\1/p' scripts/ci_install_trtllm.sh | head -1) +require "tensorrt-llm pin" "$trtllm_current" +trtllm_latest=$("${CURL[@]}" https://pypi.nvidia.com/tensorrt-llm/ \ + | grep -o 'tensorrt_llm-[0-9][^-]*' | sed 's/tensorrt_llm-//;s/rc/~rc/' | sort -uV | tail -1 | sed 's/~rc/rc/') +require "tensorrt-llm latest" "$trtllm_latest" +emit tensorrt-llm "$trtllm_current" "$trtllm_latest" + +tokenspeed_current=$(sed -n 's/.*TOKENSPEED_REF:-\([0-9a-f]*\)}.*/\1/p' scripts/ci_install_tokenspeed.sh | head -1) +require "tokenspeed pin" "$tokenspeed_current" +tokenspeed_latest=$(gh api repos/lightseekorg/tokenspeed/commits/main --jq .sha) +require "tokenspeed latest" "$tokenspeed_latest" +emit tokenspeed "$tokenspeed_current" "$tokenspeed_latest"