From 2e1ec27b3b52bc39d14338e6240a68ff2b4961f1 Mon Sep 17 00:00:00 2001 From: Simo Lin Date: Sat, 14 Mar 2026 19:36:08 -0700 Subject: [PATCH 1/2] ci: skip secret-dependent jobs on fork PRs Fork PRs don't have access to repo secrets (API keys, HF_TOKEN), causing e2e tests, vendor tests, benchmarks, and go-bindings-e2e to fail with missing credentials after maintainer approval. Add fork detection guard to all secret-dependent jobs: github.event.pull_request.head.repo.full_name == github.repository This skips these jobs on fork PRs while still running them on: - push to main - internal PRs (same repo) Affected jobs: benchmarks, e2e-1gpu-chat, e2e-1gpu-embeddings, e2e-1gpu-gateway, e2e-2gpu-responses, e2e-2gpu-pd, e2e-vendor (anthropic-messages, openai-responses, openai-realtime, xai-responses), go-bindings-e2e. Cascading jobs (e2e-2gpu-chat, e2e-4gpu-chat) auto-skip when their dependencies skip. The finish job already handles skipped results correctly (checks for "failure", not "skipped"). Fork PRs will still run: pre-commit, python-lint, unit-tests, build-wheel, python-unit-tests, go-unit-tests, grpc-proto-build-check. Signed-off-by: Simo Lin --- .github/workflows/pr-test-rust.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index fad06e85cb..682870a36c 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -286,7 +286,10 @@ jobs: benchmarks: needs: build-wheel - if: github.actor != 'dependabot[bot]' + if: >- + github.actor != 'dependabot[bot]' + && (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository) runs-on: 4-gpu-h100 timeout-minutes: 30 permissions: @@ -414,7 +417,8 @@ jobs: && needs.build-wheel.result == 'success' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' - || (needs.detect-changes.result == 'success' + || (github.event.pull_request.head.repo.full_name == github.repository + && needs.detect-changes.result == 'success' && (needs.detect-changes.outputs.common == 'true' || needs.detect-changes.outputs.chat-completions == 'true'))) strategy: @@ -451,7 +455,8 @@ jobs: && needs.build-wheel.result == 'success' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' - || (needs.detect-changes.result == 'success' + || (github.event.pull_request.head.repo.full_name == github.repository + && needs.detect-changes.result == 'success' && (needs.detect-changes.outputs.common == 'true' || needs.detect-changes.outputs.embeddings == 'true'))) uses: ./.github/workflows/e2e-gpu-job.yml @@ -473,7 +478,8 @@ jobs: && needs.build-wheel.result == 'success' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' - || (needs.detect-changes.result == 'success' + || (github.event.pull_request.head.repo.full_name == github.repository + && needs.detect-changes.result == 'success' && (needs.detect-changes.outputs.common == 'true' || needs.detect-changes.outputs.chat-completions == 'true'))) strategy: @@ -515,7 +521,8 @@ jobs: && needs.build-wheel.result == 'success' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' - || (needs.detect-changes.result == 'success' + || (github.event.pull_request.head.repo.full_name == github.repository + && needs.detect-changes.result == 'success' && (needs.detect-changes.outputs.common == 'true' || needs.detect-changes.outputs.agentic == 'true'))) uses: ./.github/workflows/e2e-gpu-job.yml @@ -536,7 +543,8 @@ jobs: && !cancelled() && needs.e2e-1gpu-gateway.result == 'success' && (github.event_name != 'pull_request' - || (needs.detect-changes.result == 'success' + || (github.event.pull_request.head.repo.full_name == github.repository + && needs.detect-changes.result == 'success' && (needs.detect-changes.outputs.common == 'true' || needs.detect-changes.outputs.chat-completions == 'true'))) strategy: @@ -580,7 +588,8 @@ jobs: && needs.build-wheel.result == 'success' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' - || (needs.detect-changes.result == 'success' + || (github.event.pull_request.head.repo.full_name == github.repository + && needs.detect-changes.result == 'success' && (needs.detect-changes.outputs.common == 'true' || needs.detect-changes.outputs.agentic == 'true'))) permissions: @@ -730,7 +739,8 @@ jobs: && needs.build-wheel.result == 'success' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' - || (needs.detect-changes.result == 'success' + || (github.event.pull_request.head.repo.full_name == github.repository + && needs.detect-changes.result == 'success' && (needs.detect-changes.outputs.common == 'true' || needs.detect-changes.outputs.go-bindings == 'true'))) runs-on: 1-gpu-h100 From 2faadc5c1dc9c58670bf0d3d98ad04dd17d2d8ac Mon Sep 17 00:00:00 2001 From: Simo Lin Date: Sat, 14 Mar 2026 19:48:45 -0700 Subject: [PATCH 2/2] ci: add label-gated integration tests for fork PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create pr-test-fork-integration.yml — a separate workflow that runs secret-dependent jobs for fork PRs via `pull_request_target`. Flow: 1. Fork PR opened → pr-test-rust.yml runs non-secret jobs (lint, build, unit tests). Secret-dependent jobs are skipped. 2. Maintainer reviews the fork's code 3. Maintainer adds `safe-to-test` label 4. This workflow triggers via pull_request_target (labeled event), checks out the PR's HEAD SHA, runs all integration tests with access to repo secrets Security: - Only triggers on `labeled` event (not synchronize) — if the contributor pushes new commits, maintainer must remove and re-add the label after reviewing the new code - Guard job checks both the label name AND that it's a fork PR (internal PRs are handled by pr-test-rust.yml) - All checkout steps use the PR's HEAD SHA, not a branch ref Also adds `checkout_ref` input to e2e-gpu-job.yml reusable workflow so callers can specify which ref to checkout (needed for pull_request_target which defaults to the base branch). Signed-off-by: Simo Lin --- .github/workflows/e2e-gpu-job.yml | 7 + .../workflows/pr-test-fork-integration.yml | 468 ++++++++++++++++++ 2 files changed, 475 insertions(+) create mode 100644 .github/workflows/pr-test-fork-integration.yml diff --git a/.github/workflows/e2e-gpu-job.yml b/.github/workflows/e2e-gpu-job.yml index a9d14d32fa..601094a142 100644 --- a/.github/workflows/e2e-gpu-job.yml +++ b/.github/workflows/e2e-gpu-job.yml @@ -39,6 +39,11 @@ on: type: boolean default: false description: "Set up Oracle DB and Brave MCP for agentic tests" + checkout_ref: + required: false + type: string + default: "" + description: "Git ref to checkout (e.g. PR head SHA for pull_request_target). Empty uses default." jobs: run: @@ -54,6 +59,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v6 + with: + ref: ${{ inputs.checkout_ref || github.sha }} # Engine-specific setup - name: Setup SGLang backend diff --git a/.github/workflows/pr-test-fork-integration.yml b/.github/workflows/pr-test-fork-integration.yml new file mode 100644 index 0000000000..a0beeb7ac8 --- /dev/null +++ b/.github/workflows/pr-test-fork-integration.yml @@ -0,0 +1,468 @@ +# Integration tests for fork PRs (requires secrets) +# +# Fork PRs cannot access repo secrets via the `pull_request` trigger. +# This workflow uses `pull_request_target` to run in the base repo context, +# giving access to secrets after a maintainer adds the `safe-to-test` label. +# +# Security model: +# 1. Contributor opens fork PR +# 2. Non-secret CI runs automatically (pr-test-rust.yml) +# 3. Maintainer reviews the fork's code +# 4. Maintainer adds `safe-to-test` label +# 5. This workflow triggers, checks out the PR code, runs with secrets +# +# IMPORTANT: Only label a PR after reviewing its code. This workflow +# executes the fork's code with access to repo secrets. + +name: "Fork PR: Integration Tests" + +on: + pull_request_target: + branches: [main] + types: [labeled] + +permissions: + contents: read + +concurrency: + group: fork-integration-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + GENAI_BENCH_IMAGE: ghcr.io/moirai-internal/genai-bench:0.0.3 + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + +jobs: + # Gate: only run for fork PRs with the safe-to-test label + guard: + if: >- + github.event.label.name == 'safe-to-test' + && github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + steps: + - run: echo "Fork PR #${{ github.event.pull_request.number }} approved for integration testing" + + build-wheel: + needs: guard + runs-on: k8s-runner-gpu + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ env.PR_HEAD_SHA }} + + - name: Setup Rust + uses: ./.github/actions/setup-rust + + - name: Build Python wheel and Go FFI library + run: | + rm -rf bindings/python/dist/ + bash scripts/ci_setup_python_venv.sh + bash scripts/ci_build_wheel.sh + + - name: Generate Python client types + run: | + source "$HOME/.cargo/env" + mkdir -p clients/openapi + cargo run -p openapi-gen -- clients/openapi/smg-openapi.yaml + pip install 'datamodel-code-generator==0.54.0' + datamodel-codegen \ + --input clients/openapi/smg-openapi.yaml \ + --input-file-type openapi \ + --output clients/python/smg_client/types/_generated.py \ + --output-model-type pydantic_v2.BaseModel \ + --use-annotated \ + --field-constraints \ + --target-python-version 3.10 \ + --collapse-root-models \ + --use-standard-collections \ + --use-union-operator + sed -i 's/class \(.*\)(Enum):/class \1(str, Enum):/' clients/python/smg_client/types/_generated.py + + - name: Build WASM test fixtures + run: | + source "$HOME/.cargo/env" + bash crates/wasm/tests/fixtures/build_fixtures.sh + + - name: Upload wheel artifact + uses: actions/upload-artifact@v7 + with: + name: smg-wheel + path: bindings/python/dist/*.whl + retention-days: 1 + + - name: Upload Go FFI library artifact + uses: actions/upload-artifact@v7 + with: + name: go-ffi-library + path: bindings/golang/target/release/libsmg_go.* + retention-days: 1 + + - name: Upload Python client types + uses: actions/upload-artifact@v7 + with: + name: python-client-types + path: clients/python/smg_client/types/_generated.py + retention-days: 1 + + - name: Upload WASM test fixtures + uses: actions/upload-artifact@v7 + with: + name: wasm-test-fixtures + path: crates/wasm/tests/fixtures/*.wasm + retention-days: 1 + if-no-files-found: ignore + + - name: Show sccache stats + if: always() + run: sccache --show-stats + + # --- GPU E2E tests --- + + e2e-1gpu-chat: + name: e2e-1gpu-chat (${{ matrix.engine }}) + needs: build-wheel + strategy: + fail-fast: false + matrix: + include: + - engine: sglang + timeout: 30 + runner: 1-gpu + test_filter: "" + - engine: vllm + timeout: 20 + runner: 1-gpu + test_filter: "-k 'not (5-grpc or 2-None-grpc or multiple_choices)'" + - engine: trtllm + timeout: 90 + runner: 1-gpu-h100 + test_filter: "" + uses: ./.github/workflows/e2e-gpu-job.yml + with: + engine: ${{ matrix.engine }} + gpu_tier: "1" + runner: ${{ matrix.runner }} + timeout: ${{ matrix.timeout }} + test_dirs: e2e_test/chat_completions + test_filter: ${{ matrix.test_filter }} + checkout_ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit + + e2e-1gpu-embeddings: + needs: build-wheel + uses: ./.github/workflows/e2e-gpu-job.yml + with: + engine: sglang + gpu_tier: "1" + runner: 1-gpu + timeout: 20 + test_dirs: e2e_test/embeddings + extra_deps: "sentence-transformers" + checkout_ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit + + e2e-1gpu-gateway: + name: e2e-1gpu-gateway (${{ matrix.engine }}) + needs: build-wheel + strategy: + fail-fast: false + matrix: + include: + - engine: sglang + timeout: 20 + - engine: vllm + timeout: 20 + uses: ./.github/workflows/e2e-gpu-job.yml + with: + engine: ${{ matrix.engine }} + gpu_tier: "1" + runner: 1-gpu + timeout: ${{ matrix.timeout }} + test_dirs: e2e_test/router + test_filter: "--ignore=e2e_test/router/test_pd_mmlu.py" + checkout_ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit + + e2e-2gpu-chat: + needs: e2e-1gpu-chat + uses: ./.github/workflows/e2e-gpu-job.yml + with: + engine: sglang + gpu_tier: "2" + runner: 2-gpu-h100 + timeout: 20 + test_dirs: e2e_test/chat_completions + checkout_ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit + + e2e-2gpu-responses: + needs: build-wheel + uses: ./.github/workflows/e2e-gpu-job.yml + with: + engine: sglang + gpu_tier: "2" + runner: 2-gpu-h100 + timeout: 30 + test_dirs: e2e_test/responses + setup_agentic_deps: true + checkout_ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit + + e2e-2gpu-pd: + name: e2e-2gpu-pd (${{ matrix.engine }}) + needs: e2e-1gpu-gateway + strategy: + fail-fast: false + matrix: + include: + - engine: sglang + timeout: 30 + - engine: vllm + timeout: 30 + uses: ./.github/workflows/e2e-gpu-job.yml + with: + engine: ${{ matrix.engine }} + gpu_tier: "2" + runner: 2-gpu-h100 + timeout: ${{ matrix.timeout }} + test_dirs: e2e_test/router + checkout_ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit + + e2e-4gpu-chat: + needs: e2e-2gpu-chat + uses: ./.github/workflows/e2e-gpu-job.yml + with: + engine: sglang + gpu_tier: "4" + runner: 4-gpu-h100 + timeout: 20 + test_dirs: e2e_test/chat_completions + checkout_ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit + + # --- Vendor E2E (CPU, cloud backends) --- + + e2e-vendor: + name: ${{ matrix.name }} + needs: build-wheel + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - name: anthropic-messages + vendor: anthropic + test_path: e2e_test/messages + timeout: 20 + setup_agentic_deps: true + + - name: openai-responses + vendor: openai + test_path: e2e_test/responses + timeout: 30 + setup_agentic_deps: true + + - name: openai-realtime + vendor: openai + test_path: e2e_test/realtime + timeout: 20 + + - name: xai-responses + vendor: xai + test_path: e2e_test/responses + timeout: 15 + setup_agentic_deps: true + + runs-on: k8s-runner-cpu + timeout-minutes: ${{ matrix.timeout }} + env: + E2E_VENDOR: ${{ matrix.vendor }} + E2E_GPU_TIER: "0" + SHOW_ROUTER_LOGS: "1" + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.PR_HEAD_SHA }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Check shared services + if: matrix.setup_agentic_deps + run: bash scripts/ci_agentic_svc_deps.sh check --oracle oracle-db --brave brave-search-mcp + + - name: Download wheel artifact + uses: actions/download-artifact@v8 + with: + name: smg-wheel + path: wheel/ + + - name: Download WASM test fixtures + uses: actions/download-artifact@v8 + with: + name: wasm-test-fixtures + path: crates/wasm/tests/fixtures/ + continue-on-error: true + + - name: Download Python client types + uses: actions/download-artifact@v8 + with: + name: python-client-types + path: clients/python/smg_client/types/ + + - name: Install wheel and test dependencies + run: | + pip uninstall -y smg || true + pip install wheel/*.whl + bash scripts/ci_install_e2e_deps.sh + + - name: Setup Oracle + if: matrix.setup_agentic_deps + run: | + bash scripts/ci_agentic_svc_deps.sh setup-oracle-client + bash scripts/ci_agentic_svc_deps.sh create-oracle-user oracle-db + bash scripts/ci_agentic_svc_deps.sh create-oracle-flyway-user oracle-db + + - name: Run E2E tests + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_API_KEY }} + BRAVE_MCP_HOST: ${{ matrix.setup_agentic_deps && 'brave-search-mcp' || '' }} + run: | + ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ matrix.test_path }} \ + -m "not external" \ + --reruns 2 --reruns-delay 5 \ + -s -vv + + - name: Cleanup Oracle test users + if: always() && matrix.setup_agentic_deps + run: | + bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-flyway-user oracle-db + bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-user oracle-db + + # --- Benchmarks --- + + benchmarks: + needs: build-wheel + runs-on: 4-gpu-h100 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.PR_HEAD_SHA }} + + - name: Setup SGLang backend + uses: ./.github/actions/setup-sglang + + - name: Download wheel artifact + uses: actions/download-artifact@v8 + with: + name: smg-wheel + path: wheel/ + + - name: Download WASM test fixtures + uses: actions/download-artifact@v8 + with: + name: wasm-test-fixtures + path: crates/wasm/tests/fixtures/ + continue-on-error: true + + - name: Download Python client types + uses: actions/download-artifact@v8 + with: + name: python-client-types + path: clients/python/smg_client/types/ + + - name: Install wheel and test dependencies + run: | + pip uninstall -y smg || true + pip install wheel/*.whl + bash scripts/ci_install_e2e_deps.sh + + - name: Pull genai-bench image + run: docker pull ${{ env.GENAI_BENCH_IMAGE }} + + - name: Run benchmarks + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + ROUTER_LOCAL_MODEL_PATH: /home/ubuntu/models + run: | + bash scripts/ci_killall_sglang.sh "nuk_gpus" + SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1 pytest e2e_test/benchmarks \ + --ignore=e2e_test/benchmarks/test_go_bindings_perf.py \ + --ignore=e2e_test/benchmarks/test_nightly_perf.py \ + -s -vv + + # --- Go bindings E2E --- + + go-bindings-e2e: + needs: build-wheel + runs-on: 1-gpu-h100 + timeout-minutes: 20 + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.PR_HEAD_SHA }} + + - name: Setup SGLang backend + uses: ./.github/actions/setup-sglang + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version: '1.24' + cache: true + cache-dependency-path: bindings/golang/go.sum + + - name: Download Go FFI library + uses: actions/download-artifact@v8 + with: + name: go-ffi-library + path: bindings/golang/target/release/ + + - name: Verify Go FFI library + run: ls -la bindings/golang/target/release/libsmg_go.* + + - name: Download wheel artifact + uses: actions/download-artifact@v8 + with: + name: smg-wheel + path: wheel/ + + - name: Download Python client types + uses: actions/download-artifact@v8 + with: + name: python-client-types + path: clients/python/smg_client/types/ + + - name: Install wheel and test dependencies + run: | + pip uninstall -y smg || true + pip install wheel/*.whl + bash scripts/ci_install_e2e_deps.sh + + - name: Run Go OAI server E2E tests + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + bash scripts/ci_killall_sglang.sh "nuk_gpus" + export CGO_LDFLAGS="-L$(pwd)/bindings/golang/target/release" + export LD_LIBRARY_PATH="$(pwd)/bindings/golang/target/release:$LD_LIBRARY_PATH" + SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1 ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" \ + pytest --reruns 2 --reruns-delay 5 e2e_test/bindings_go -s -vv