From afb5c032e4e2f899a5e1ee7afa8e1d2bf7bf5fde Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 12:09:44 -0800 Subject: [PATCH 01/12] ci: skip irrelevant E2E jobs on PRs and move labeler off self-hosted runner Signed-off-by: Chang Su --- .github/actions/run-e2e/action.yml | 128 ++++++++++++++ .github/workflows/labeler.yml | 6 +- .github/workflows/pr-test-rust.yml | 262 +++++++++++++++++------------ 3 files changed, 291 insertions(+), 105 deletions(-) create mode 100644 .github/actions/run-e2e/action.yml diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml new file mode 100644 index 0000000000..e3f35a33f7 --- /dev/null +++ b/.github/actions/run-e2e/action.yml @@ -0,0 +1,128 @@ +name: 'Run E2E Tests' +description: 'Shared steps for all gateway E2E test jobs.' + +inputs: + # Backend selection + setup_vllm: + default: 'false' + setup_trtllm: + default: 'false' + # Test configuration + test_dirs: + required: true + extra_deps: + default: '' + env_vars: + default: '' + reruns: + default: '' + parallel_opts: + default: '' + ignore_opts: + default: '' + test_filter: + default: '' + # Optional features + setup_agentic_deps: + default: 'false' + upload_benchmarks: + default: 'false' + genai_bench_image: + default: '' + # Secrets (must be passed explicitly to composite actions) + hf_token: + default: '' + openai_api_key: + default: '' + xai_api_key: + default: '' + anthropic_api_key: + default: '' + +runs: + using: 'composite' + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Check shared services + if: inputs.setup_agentic_deps == 'true' + shell: bash + run: bash scripts/ci_agentic_svc_deps.sh check --oracle oracle-db --brave brave-search-mcp + + - name: Setup vLLM backend + if: inputs.setup_vllm == 'true' + uses: ./.github/actions/setup-vllm + + - name: Setup TRT-LLM backend + if: inputs.setup_trtllm == 'true' + uses: ./.github/actions/setup-trtllm + + - name: Setup SGLang backend + if: inputs.setup_vllm != 'true' && inputs.setup_trtllm != 'true' + uses: ./.github/actions/setup-sglang + + - name: Download wheel artifact + uses: actions/download-artifact@v8 + with: + name: smg-wheel + path: wheel/ + + - name: Download WASM test fixtures + uses: actions/download-artifact@v8 + with: + name: wasm-test-fixtures + path: wasm/tests/fixtures/ + continue-on-error: true + + - name: Download Python client types + uses: actions/download-artifact@v8 + with: + name: python-client-types + path: clients/python/smg_client/types/ + + - name: Install wheel and test dependencies + shell: bash + run: | + pip uninstall -y smg || true + pip install wheel/*.whl + bash scripts/ci_install_e2e_deps.sh ${{ inputs.extra_deps }} + + - name: Pull genai-bench image + if: inputs.upload_benchmarks == 'true' + shell: bash + run: docker pull ${{ inputs.genai_bench_image }} + + - name: Setup Oracle + if: inputs.setup_agentic_deps == 'true' + shell: bash + run: | + bash scripts/ci_agentic_svc_deps.sh setup-oracle-client + bash scripts/ci_agentic_svc_deps.sh create-oracle-user oracle-db + bash scripts/ci_agentic_svc_deps.sh create-oracle-flyway-user oracle-db + + - name: Run E2E tests + shell: bash + env: + HF_TOKEN: ${{ inputs.hf_token }} + OPENAI_API_KEY: ${{ inputs.openai_api_key }} + XAI_API_KEY: ${{ inputs.xai_api_key }} + ANTHROPIC_API_KEY: ${{ inputs.anthropic_api_key }} + BRAVE_MCP_HOST: ${{ inputs.setup_agentic_deps == 'true' && 'brave-search-mcp' || '' }} + run: | + bash scripts/ci_killall_sglang.sh "nuk_gpus" + ${{ inputs.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ inputs.reruns }} ${{ inputs.parallel_opts }} ${{ inputs.ignore_opts }} ${{ inputs.test_dirs }} ${{ inputs.test_filter }} -s -vv -o log_cli=true --log-cli-level=INFO + + - name: Cleanup Oracle test users + if: always() && inputs.setup_agentic_deps == 'true' + shell: bash + run: | + bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-flyway-user oracle-db + bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-user oracle-db + + - name: Upload benchmark results + if: inputs.upload_benchmarks == 'true' + uses: actions/upload-artifact@v7 + with: + name: genai-bench-results-all-policies + path: benchmark_**/ diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 70e40094bd..9f7b973c76 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -4,13 +4,17 @@ on: pull_request_target: types: [opened, synchronize, reopened] +concurrency: + group: labeler-${{ github.head_ref }} + cancel-in-progress: true + permissions: contents: read pull-requests: write jobs: labeler: - runs-on: k8s-runner-cpu + runs-on: ubuntu-latest steps: - name: Label PR based on changed files uses: actions/labeler@v6 diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index deab316e60..a7021a6308 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -282,7 +282,50 @@ jobs: if: always() run: sccache --show-stats - gateway-e2e: + detect-changes: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + permissions: + contents: read + outputs: + common: ${{ steps.filter.outputs.common }} + chat-completions: ${{ steps.filter.outputs.chat-completions }} + agentic: ${{ steps.filter.outputs.agentic }} + steps: + - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + common: + - 'model_gateway/**' + - 'protocols/**' + - 'bindings/**' + - 'e2e_test/conftest.py' + - 'e2e_test/infra/**' + - 'e2e_test/fixtures/**' + - 'Cargo.lock' + chat-completions: + - 'tokenizer/**' + - 'tool_parser/**' + - 'reasoning_parser/**' + - 'multimodal/**' + - 'grpc_client/**' + - 'e2e_test/chat_completions/**' + agentic: + - 'mcp/**' + - 'workflow/**' + - 'data_connector/**' + - 'tool_parser/**' + - 'tokenizer/**' + - 'auth/**' + - 'e2e_test/responses/**' + - 'e2e_test/messages/**' + - 'scripts/ci_agentic_svc_deps.sh' + + # --- E2E jobs split by change scope --- + + e2e-always: name: ${{ matrix.name }} needs: build-wheel if: github.actor != 'dependabot[bot]' @@ -298,20 +341,10 @@ jobs: extra_deps: "" env_vars: "" reruns: "" - upload_benchmarks: true + upload_benchmarks: 'true' parallel_opts: "" # No parallel for benchmarks (performance measurement) ignore_opts: "--ignore=e2e_test/benchmarks/test_go_bindings_perf.py --ignore=e2e_test/benchmarks/test_nightly_perf.py" # Go and nightly benchmarks run in dedicated jobs runner: 4-gpu-h100 - - name: agentic-apis - timeout: 30 - test_dirs: "e2e_test/responses e2e_test/messages" - extra_deps: "" - env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" - reruns: "--reruns 2 --reruns-delay 5" - setup_agentic_deps: true - parallel_opts: "" - ignore_opts: "" - test_filter: "-m 'not external'" - name: e2e timeout: 20 test_dirs: "e2e_test/router e2e_test/embeddings" @@ -321,6 +354,54 @@ jobs: reruns: "--reruns 2 --reruns-delay 5" parallel_opts: "--workers 1 --tests-per-worker 4" # Thread-based parallelism ignore_opts: "" + - name: vllm-pd + timeout: 15 + test_dirs: "e2e_test/router/test_pd_mmlu.py" + extra_deps: "" + env_vars: "E2E_RUNTIME=vllm SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" + reruns: "--reruns 2 --reruns-delay 5" + parallel_opts: "" + test_filter: "" + setup_vllm: 'true' + ignore_opts: "" + runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} + timeout-minutes: ${{ matrix.timeout }} + steps: + - uses: ./.github/actions/run-e2e + with: + setup_vllm: ${{ matrix.setup_vllm }} + setup_trtllm: ${{ matrix.setup_trtllm }} + test_dirs: ${{ matrix.test_dirs }} + extra_deps: ${{ matrix.extra_deps }} + env_vars: ${{ matrix.env_vars }} + reruns: ${{ matrix.reruns }} + parallel_opts: ${{ matrix.parallel_opts }} + ignore_opts: ${{ matrix.ignore_opts }} + test_filter: ${{ matrix.test_filter }} + setup_agentic_deps: ${{ matrix.setup_agentic_deps }} + upload_benchmarks: ${{ matrix.upload_benchmarks }} + genai_bench_image: ${{ env.GENAI_BENCH_IMAGE }} + hf_token: ${{ secrets.HF_TOKEN }} + openai_api_key: ${{ secrets.OPENAI_API_KEY }} + xai_api_key: ${{ secrets.XAI_API_KEY }} + anthropic_api_key: ${{ secrets.CLAUDE_API_KEY }} + + e2e-chat-completions: + name: ${{ matrix.name }} + needs: [build-wheel, detect-changes] + if: >- + !cancelled() + && needs.build-wheel.result == 'success' + && github.actor != 'dependabot[bot]' + && (github.event_name != 'pull_request' + || needs.detect-changes.outputs.common == 'true' + || needs.detect-changes.outputs.chat-completions == 'true') + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: - name: chat-completions-sglang timeout: 20 test_dirs: "e2e_test/chat_completions" @@ -340,17 +421,7 @@ jobs: # TODO: Remove filter when vLLM supports logprobs and n>1 with greedy sampling # Excludes: 5-grpc (logprobs=5), 2-None-grpc (n=2 with no logprobs), multiple_choices (n=2 tests) test_filter: "-k 'not (5-grpc or 2-None-grpc or multiple_choices)'" - setup_vllm: true - ignore_opts: "" - - name: vllm-pd - timeout: 15 - test_dirs: "e2e_test/router/test_pd_mmlu.py" - extra_deps: "" - env_vars: "E2E_RUNTIME=vllm SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" - reruns: "--reruns 2 --reruns-delay 5" - parallel_opts: "" - test_filter: "" - setup_vllm: true + setup_vllm: 'true' ignore_opts: "" - name: chat-completions-trtllm timeout: 60 @@ -360,90 +431,71 @@ jobs: reruns: "--reruns 2 --reruns-delay 5" parallel_opts: "--workers 1 --tests-per-worker 4" test_filter: "" - setup_trtllm: true + setup_trtllm: 'true' ignore_opts: "" runner: 4-gpu-h100 runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Check shared services - if: matrix.setup_agentic_deps - run: bash scripts/ci_agentic_svc_deps.sh check --oracle oracle-db --brave brave-search-mcp - - - name: Setup vLLM backend - if: matrix.setup_vllm - uses: ./.github/actions/setup-vllm - - - name: Setup TRT-LLM backend - if: matrix.setup_trtllm - uses: ./.github/actions/setup-trtllm - - - name: Setup SGLang backend - if: ${{ !matrix.setup_vllm && !matrix.setup_trtllm }} - uses: ./.github/actions/setup-sglang - - - name: Download wheel artifact - uses: actions/download-artifact@v8 - with: - name: smg-wheel - path: wheel/ - - - name: Download WASM test fixtures - uses: actions/download-artifact@v8 - with: - name: wasm-test-fixtures - path: wasm/tests/fixtures/ - continue-on-error: true - - - name: Download Python client types - uses: actions/download-artifact@v8 + - uses: ./.github/actions/run-e2e with: - name: python-client-types - path: clients/python/smg_client/types/ - - - name: Install wheel and test dependencies - run: | - pip uninstall -y smg || true - pip install wheel/*.whl - bash scripts/ci_install_e2e_deps.sh ${{ matrix.extra_deps }} - - - name: Pull genai-bench image - if: matrix.upload_benchmarks - run: docker pull ${{ env.GENAI_BENCH_IMAGE }} - - - name: Setup Oracle - if: matrix.setup_agentic_deps - run: | - bash scripts/ci_agentic_svc_deps.sh setup-oracle-client - bash scripts/ci_agentic_svc_deps.sh create-oracle-user oracle-db - bash scripts/ci_agentic_svc_deps.sh create-oracle-flyway-user oracle-db - - - name: Run E2E tests - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_API_KEY }} - BRAVE_MCP_HOST: ${{ matrix.setup_agentic_deps && 'brave-search-mcp' || '' }} - run: | - bash scripts/ci_killall_sglang.sh "nuk_gpus" - ${{ matrix.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.ignore_opts }} ${{ matrix.test_dirs }} ${{ matrix.test_filter }} -s -vv -o log_cli=true --log-cli-level=INFO - - - name: Cleanup Oracle test users - if: always() && matrix.setup_agentic_deps - run: | - bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-flyway-user oracle-db - bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-user oracle-db - - - name: Upload benchmark results - if: matrix.upload_benchmarks && success() - uses: actions/upload-artifact@v7 + setup_vllm: ${{ matrix.setup_vllm }} + setup_trtllm: ${{ matrix.setup_trtllm }} + test_dirs: ${{ matrix.test_dirs }} + extra_deps: ${{ matrix.extra_deps }} + env_vars: ${{ matrix.env_vars }} + reruns: ${{ matrix.reruns }} + parallel_opts: ${{ matrix.parallel_opts }} + ignore_opts: ${{ matrix.ignore_opts }} + test_filter: ${{ matrix.test_filter }} + hf_token: ${{ secrets.HF_TOKEN }} + openai_api_key: ${{ secrets.OPENAI_API_KEY }} + xai_api_key: ${{ secrets.XAI_API_KEY }} + anthropic_api_key: ${{ secrets.CLAUDE_API_KEY }} + + e2e-agentic: + name: ${{ matrix.name }} + needs: [build-wheel, detect-changes] + if: >- + !cancelled() + && needs.build-wheel.result == 'success' + && github.actor != 'dependabot[bot]' + && (github.event_name != 'pull_request' + || needs.detect-changes.outputs.common == 'true' + || needs.detect-changes.outputs.agentic == 'true') + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - name: agentic-apis + timeout: 30 + test_dirs: "e2e_test/responses e2e_test/messages" + extra_deps: "" + env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" + reruns: "--reruns 2 --reruns-delay 5" + setup_agentic_deps: 'true' + parallel_opts: "" + ignore_opts: "" + test_filter: "-m 'not external'" + runs-on: k8s-runner-gpu + timeout-minutes: ${{ matrix.timeout }} + steps: + - uses: ./.github/actions/run-e2e with: - name: genai-bench-results-all-policies - path: benchmark_**/ + test_dirs: ${{ matrix.test_dirs }} + extra_deps: ${{ matrix.extra_deps }} + env_vars: ${{ matrix.env_vars }} + reruns: ${{ matrix.reruns }} + parallel_opts: ${{ matrix.parallel_opts }} + ignore_opts: ${{ matrix.ignore_opts }} + test_filter: ${{ matrix.test_filter }} + setup_agentic_deps: ${{ matrix.setup_agentic_deps }} + hf_token: ${{ secrets.HF_TOKEN }} + openai_api_key: ${{ secrets.OPENAI_API_KEY }} + xai_api_key: ${{ secrets.XAI_API_KEY }} + anthropic_api_key: ${{ secrets.CLAUDE_API_KEY }} go-unit-tests: name: go-unit-tests @@ -609,9 +661,9 @@ jobs: path: benchmark_go_bindings/ finish: - needs: [pre-commit, python-lint, grpc-proto-build-check, build-wheel, python-unit-tests, unit-tests, gateway-e2e, go-unit-tests, go-bindings-e2e] + needs: [pre-commit, python-lint, grpc-proto-build-check, build-wheel, python-unit-tests, unit-tests, detect-changes, e2e-always, e2e-chat-completions, e2e-agentic, go-unit-tests, go-bindings-e2e] if: always() - runs-on: k8s-runner-cpu + runs-on: ubuntu-latest permissions: {} steps: - name: Check CI result @@ -622,7 +674,9 @@ jobs: "${{ needs.build-wheel.result }}" == "failure" || \ "${{ needs.python-unit-tests.result }}" == "failure" || \ "${{ needs.unit-tests.result }}" == "failure" || \ - "${{ needs.gateway-e2e.result }}" == "failure" || \ + "${{ needs.e2e-always.result }}" == "failure" || \ + "${{ needs.e2e-chat-completions.result }}" == "failure" || \ + "${{ needs.e2e-agentic.result }}" == "failure" || \ "${{ needs.go-unit-tests.result }}" == "failure" || \ "${{ needs.go-bindings-e2e.result }}" == "failure" ]]; then echo "One or more jobs failed" @@ -632,7 +686,7 @@ jobs: fi summarize-benchmarks: - needs: [gateway-e2e] + needs: [e2e-always] runs-on: k8s-runner-cpu if: success() permissions: From e2c6541b3543f931259248ead826a9fa42cf2cdb Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 12:13:48 -0800 Subject: [PATCH 02/12] ci: revert labeler and finish jobs back to k8s-runner-cpu Signed-off-by: Chang Su --- .github/workflows/labeler.yml | 2 +- .github/workflows/pr-test-rust.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 9f7b973c76..5b308fe952 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -14,7 +14,7 @@ permissions: jobs: labeler: - runs-on: ubuntu-latest + runs-on: k8s-runner-cpu steps: - name: Label PR based on changed files uses: actions/labeler@v6 diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index a7021a6308..03756e4fe3 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -663,7 +663,7 @@ jobs: finish: needs: [pre-commit, python-lint, grpc-proto-build-check, build-wheel, python-unit-tests, unit-tests, detect-changes, e2e-always, e2e-chat-completions, e2e-agentic, go-unit-tests, go-bindings-e2e] if: always() - runs-on: ubuntu-latest + runs-on: k8s-runner-cpu permissions: {} steps: - name: Check CI result From d572b665f18f7c3c6f75aa01a81645f440646b2a Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 12:39:37 -0800 Subject: [PATCH 03/12] =?UTF-8?q?ci:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20explicit=20matrix=20defaults,=20permissions,=20and?= =?UTF-8?q?=20CI=20action=20filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Chang Su --- .github/workflows/pr-test-rust.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 03756e4fe3..dfea928010 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -287,6 +287,7 @@ jobs: if: github.event_name == 'pull_request' permissions: contents: read + pull-requests: read outputs: common: ${{ steps.filter.outputs.common }} chat-completions: ${{ steps.filter.outputs.chat-completions }} @@ -305,6 +306,7 @@ jobs: - 'e2e_test/infra/**' - 'e2e_test/fixtures/**' - 'Cargo.lock' + - '.github/actions/**' chat-completions: - 'tokenizer/**' - 'tool_parser/**' @@ -341,6 +343,9 @@ jobs: extra_deps: "" env_vars: "" reruns: "" + setup_vllm: 'false' + setup_trtllm: 'false' + setup_agentic_deps: 'false' upload_benchmarks: 'true' parallel_opts: "" # No parallel for benchmarks (performance measurement) ignore_opts: "--ignore=e2e_test/benchmarks/test_go_bindings_perf.py --ignore=e2e_test/benchmarks/test_nightly_perf.py" # Go and nightly benchmarks run in dedicated jobs @@ -352,6 +357,10 @@ jobs: extra_deps: "pytest-parallel py sentence-transformers" env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" reruns: "--reruns 2 --reruns-delay 5" + setup_vllm: 'false' + setup_trtllm: 'false' + setup_agentic_deps: 'false' + upload_benchmarks: 'false' parallel_opts: "--workers 1 --tests-per-worker 4" # Thread-based parallelism ignore_opts: "" - name: vllm-pd @@ -363,6 +372,9 @@ jobs: parallel_opts: "" test_filter: "" setup_vllm: 'true' + setup_trtllm: 'false' + setup_agentic_deps: 'false' + upload_benchmarks: 'false' ignore_opts: "" runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} @@ -410,6 +422,8 @@ jobs: reruns: "--reruns 2 --reruns-delay 5" parallel_opts: "--workers 1 --tests-per-worker 4" test_filter: "" + setup_vllm: 'false' + setup_trtllm: 'false' ignore_opts: "" - name: chat-completions-vllm timeout: 20 @@ -422,6 +436,7 @@ jobs: # Excludes: 5-grpc (logprobs=5), 2-None-grpc (n=2 with no logprobs), multiple_choices (n=2 tests) test_filter: "-k 'not (5-grpc or 2-None-grpc or multiple_choices)'" setup_vllm: 'true' + setup_trtllm: 'false' ignore_opts: "" - name: chat-completions-trtllm timeout: 60 @@ -431,6 +446,7 @@ jobs: reruns: "--reruns 2 --reruns-delay 5" parallel_opts: "--workers 1 --tests-per-worker 4" test_filter: "" + setup_vllm: 'false' setup_trtllm: 'true' ignore_opts: "" runner: 4-gpu-h100 From 3b3d6e6dd29863e241681d552e211d3c83cd9a8d Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 12:46:59 -0800 Subject: [PATCH 04/12] ci: add checkout before local composite action reference Signed-off-by: Chang Su --- .github/actions/run-e2e/action.yml | 3 --- .github/workflows/pr-test-rust.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml index e3f35a33f7..2a98b09213 100644 --- a/.github/actions/run-e2e/action.yml +++ b/.github/actions/run-e2e/action.yml @@ -42,9 +42,6 @@ inputs: runs: using: 'composite' steps: - - name: Checkout code - uses: actions/checkout@v6 - - name: Check shared services if: inputs.setup_agentic_deps == 'true' shell: bash diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index dfea928010..ff339d3c4d 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -379,6 +379,7 @@ jobs: runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} steps: + - uses: actions/checkout@v6 - uses: ./.github/actions/run-e2e with: setup_vllm: ${{ matrix.setup_vllm }} @@ -453,6 +454,7 @@ jobs: runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} steps: + - uses: actions/checkout@v6 - uses: ./.github/actions/run-e2e with: setup_vllm: ${{ matrix.setup_vllm }} @@ -498,6 +500,7 @@ jobs: runs-on: k8s-runner-gpu timeout-minutes: ${{ matrix.timeout }} steps: + - uses: actions/checkout@v6 - uses: ./.github/actions/run-e2e with: test_dirs: ${{ matrix.test_dirs }} From 24e865af276636dc8b0d3b542c55e5471d6d1a96 Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 12:49:42 -0800 Subject: [PATCH 05/12] ci: fix skipped detect-changes blocking non-PR runs, add workflow file to common filter Signed-off-by: Chang Su --- .github/workflows/pr-test-rust.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index ff339d3c4d..9f25935cc0 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -307,6 +307,7 @@ jobs: - 'e2e_test/fixtures/**' - 'Cargo.lock' - '.github/actions/**' + - '.github/workflows/pr-test-rust.yml' chat-completions: - 'tokenizer/**' - 'tool_parser/**' @@ -403,12 +404,14 @@ jobs: name: ${{ matrix.name }} needs: [build-wheel, detect-changes] if: >- - !cancelled() + always() + && !cancelled() && needs.build-wheel.result == 'success' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' - || needs.detect-changes.outputs.common == 'true' - || needs.detect-changes.outputs.chat-completions == 'true') + || (needs.detect-changes.result == 'success' + && (needs.detect-changes.outputs.common == 'true' + || needs.detect-changes.outputs.chat-completions == 'true'))) permissions: contents: read strategy: @@ -475,12 +478,14 @@ jobs: name: ${{ matrix.name }} needs: [build-wheel, detect-changes] if: >- - !cancelled() + always() + && !cancelled() && needs.build-wheel.result == 'success' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' - || needs.detect-changes.outputs.common == 'true' - || needs.detect-changes.outputs.agentic == 'true') + || (needs.detect-changes.result == 'success' + && (needs.detect-changes.outputs.common == 'true' + || needs.detect-changes.outputs.agentic == 'true'))) permissions: contents: read strategy: From 61f2386c953085904adc6ff3252f539bbb2793b8 Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 12:51:05 -0800 Subject: [PATCH 06/12] ci: check detect-changes result in finish job Signed-off-by: Chang Su --- .github/workflows/pr-test-rust.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 9f25935cc0..4a6ccf46be 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -698,6 +698,7 @@ jobs: "${{ needs.build-wheel.result }}" == "failure" || \ "${{ needs.python-unit-tests.result }}" == "failure" || \ "${{ needs.unit-tests.result }}" == "failure" || \ + "${{ needs.detect-changes.result }}" == "failure" || \ "${{ needs.e2e-always.result }}" == "failure" || \ "${{ needs.e2e-chat-completions.result }}" == "failure" || \ "${{ needs.e2e-agentic.result }}" == "failure" || \ From d236e442d3fc773a2760cc493a7ef98dae9fab0c Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 13:02:53 -0800 Subject: [PATCH 07/12] ci: add backend setup scripts to detect-changes filters Signed-off-by: Chang Su --- .github/workflows/pr-test-rust.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 4a6ccf46be..d82c7bc13c 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -308,6 +308,9 @@ jobs: - 'Cargo.lock' - '.github/actions/**' - '.github/workflows/pr-test-rust.yml' + - 'scripts/ci_setup_python_venv.sh' + - 'scripts/ci_install_sglang.sh' + - 'scripts/ci_build_wheel.sh' chat-completions: - 'tokenizer/**' - 'tool_parser/**' @@ -315,6 +318,8 @@ jobs: - 'multimodal/**' - 'grpc_client/**' - 'e2e_test/chat_completions/**' + - 'scripts/ci_install_vllm.sh' + - 'scripts/ci_install_trtllm.sh' agentic: - 'mcp/**' - 'workflow/**' From b69b7e62488d604d97e394d983cae36f15b0a1e1 Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 13:19:19 -0800 Subject: [PATCH 08/12] =?UTF-8?q?ci:=20refine=20detect-changes=20filters?= =?UTF-8?q?=20=E2=80=94=20move=20shared=20crates=20to=20common,=20trim=20a?= =?UTF-8?q?gentic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Chang Su --- .github/workflows/pr-test-rust.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index d82c7bc13c..8afa29a6c1 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -311,9 +311,9 @@ jobs: - 'scripts/ci_setup_python_venv.sh' - 'scripts/ci_install_sglang.sh' - 'scripts/ci_build_wheel.sh' - chat-completions: - 'tokenizer/**' - 'tool_parser/**' + chat-completions: - 'reasoning_parser/**' - 'multimodal/**' - 'grpc_client/**' @@ -322,11 +322,7 @@ jobs: - 'scripts/ci_install_trtllm.sh' agentic: - 'mcp/**' - - 'workflow/**' - 'data_connector/**' - - 'tool_parser/**' - - 'tokenizer/**' - - 'auth/**' - 'e2e_test/responses/**' - 'e2e_test/messages/**' - 'scripts/ci_agentic_svc_deps.sh' From 5a03752933ef6df7572c4553d0c2267e62a9baee Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 13:25:44 -0800 Subject: [PATCH 09/12] ci: move vllm-pd from e2e-always to e2e-chat-completions Signed-off-by: Chang Su --- .github/workflows/pr-test-rust.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 8afa29a6c1..e10ea7c0ad 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -318,6 +318,7 @@ jobs: - 'multimodal/**' - 'grpc_client/**' - 'e2e_test/chat_completions/**' + - 'e2e_test/router/test_pd_mmlu.py' - 'scripts/ci_install_vllm.sh' - 'scripts/ci_install_trtllm.sh' agentic: @@ -365,19 +366,6 @@ jobs: upload_benchmarks: 'false' parallel_opts: "--workers 1 --tests-per-worker 4" # Thread-based parallelism ignore_opts: "" - - name: vllm-pd - timeout: 15 - test_dirs: "e2e_test/router/test_pd_mmlu.py" - extra_deps: "" - env_vars: "E2E_RUNTIME=vllm SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" - reruns: "--reruns 2 --reruns-delay 5" - parallel_opts: "" - test_filter: "" - setup_vllm: 'true' - setup_trtllm: 'false' - setup_agentic_deps: 'false' - upload_benchmarks: 'false' - ignore_opts: "" runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} steps: @@ -455,6 +443,17 @@ jobs: setup_trtllm: 'true' ignore_opts: "" runner: 4-gpu-h100 + - name: vllm-pd + timeout: 15 + test_dirs: "e2e_test/router/test_pd_mmlu.py" + extra_deps: "" + env_vars: "E2E_RUNTIME=vllm SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" + reruns: "--reruns 2 --reruns-delay 5" + parallel_opts: "" + test_filter: "" + setup_vllm: 'true' + setup_trtllm: 'false' + ignore_opts: "" runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} steps: From 6f4185dc78a54bb7f18f6d2ccd38af448cc6ea4b Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 13:27:33 -0800 Subject: [PATCH 10/12] ci: use PR number in labeler concurrency group for fork safety Signed-off-by: Chang Su --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 5b308fe952..c144295125 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -5,7 +5,7 @@ on: types: [opened, synchronize, reopened] concurrency: - group: labeler-${{ github.head_ref }} + group: labeler-${{ github.event.pull_request.number }} cancel-in-progress: true permissions: From a464e28d55d1a65edec18f6aed06ba66545e19cb Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 14:00:27 -0800 Subject: [PATCH 11/12] ci: inline E2E steps back into jobs, remove composite action Reverts the composite action approach in favor of inline steps so each step appears as a separate top-level entry in CI logs. Signed-off-by: Chang Su --- .github/actions/run-e2e/action.yml | 125 --------------- .github/workflows/pr-test-rust.yml | 235 +++++++++++++++++++++++------ 2 files changed, 185 insertions(+), 175 deletions(-) delete mode 100644 .github/actions/run-e2e/action.yml diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml deleted file mode 100644 index 2a98b09213..0000000000 --- a/.github/actions/run-e2e/action.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: 'Run E2E Tests' -description: 'Shared steps for all gateway E2E test jobs.' - -inputs: - # Backend selection - setup_vllm: - default: 'false' - setup_trtllm: - default: 'false' - # Test configuration - test_dirs: - required: true - extra_deps: - default: '' - env_vars: - default: '' - reruns: - default: '' - parallel_opts: - default: '' - ignore_opts: - default: '' - test_filter: - default: '' - # Optional features - setup_agentic_deps: - default: 'false' - upload_benchmarks: - default: 'false' - genai_bench_image: - default: '' - # Secrets (must be passed explicitly to composite actions) - hf_token: - default: '' - openai_api_key: - default: '' - xai_api_key: - default: '' - anthropic_api_key: - default: '' - -runs: - using: 'composite' - steps: - - name: Check shared services - if: inputs.setup_agentic_deps == 'true' - shell: bash - run: bash scripts/ci_agentic_svc_deps.sh check --oracle oracle-db --brave brave-search-mcp - - - name: Setup vLLM backend - if: inputs.setup_vllm == 'true' - uses: ./.github/actions/setup-vllm - - - name: Setup TRT-LLM backend - if: inputs.setup_trtllm == 'true' - uses: ./.github/actions/setup-trtllm - - - name: Setup SGLang backend - if: inputs.setup_vllm != 'true' && inputs.setup_trtllm != 'true' - uses: ./.github/actions/setup-sglang - - - name: Download wheel artifact - uses: actions/download-artifact@v8 - with: - name: smg-wheel - path: wheel/ - - - name: Download WASM test fixtures - uses: actions/download-artifact@v8 - with: - name: wasm-test-fixtures - path: wasm/tests/fixtures/ - continue-on-error: true - - - name: Download Python client types - uses: actions/download-artifact@v8 - with: - name: python-client-types - path: clients/python/smg_client/types/ - - - name: Install wheel and test dependencies - shell: bash - run: | - pip uninstall -y smg || true - pip install wheel/*.whl - bash scripts/ci_install_e2e_deps.sh ${{ inputs.extra_deps }} - - - name: Pull genai-bench image - if: inputs.upload_benchmarks == 'true' - shell: bash - run: docker pull ${{ inputs.genai_bench_image }} - - - name: Setup Oracle - if: inputs.setup_agentic_deps == 'true' - shell: bash - run: | - bash scripts/ci_agentic_svc_deps.sh setup-oracle-client - bash scripts/ci_agentic_svc_deps.sh create-oracle-user oracle-db - bash scripts/ci_agentic_svc_deps.sh create-oracle-flyway-user oracle-db - - - name: Run E2E tests - shell: bash - env: - HF_TOKEN: ${{ inputs.hf_token }} - OPENAI_API_KEY: ${{ inputs.openai_api_key }} - XAI_API_KEY: ${{ inputs.xai_api_key }} - ANTHROPIC_API_KEY: ${{ inputs.anthropic_api_key }} - BRAVE_MCP_HOST: ${{ inputs.setup_agentic_deps == 'true' && 'brave-search-mcp' || '' }} - run: | - bash scripts/ci_killall_sglang.sh "nuk_gpus" - ${{ inputs.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ inputs.reruns }} ${{ inputs.parallel_opts }} ${{ inputs.ignore_opts }} ${{ inputs.test_dirs }} ${{ inputs.test_filter }} -s -vv -o log_cli=true --log-cli-level=INFO - - - name: Cleanup Oracle test users - if: always() && inputs.setup_agentic_deps == 'true' - shell: bash - run: | - bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-flyway-user oracle-db - bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-user oracle-db - - - name: Upload benchmark results - if: inputs.upload_benchmarks == 'true' - uses: actions/upload-artifact@v7 - with: - name: genai-bench-results-all-policies - path: benchmark_**/ diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index e10ea7c0ad..a141754d90 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -369,25 +369,84 @@ jobs: runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} steps: - - uses: actions/checkout@v6 - - uses: ./.github/actions/run-e2e - with: - setup_vllm: ${{ matrix.setup_vllm }} - setup_trtllm: ${{ matrix.setup_trtllm }} - test_dirs: ${{ matrix.test_dirs }} - extra_deps: ${{ matrix.extra_deps }} - env_vars: ${{ matrix.env_vars }} - reruns: ${{ matrix.reruns }} - parallel_opts: ${{ matrix.parallel_opts }} - ignore_opts: ${{ matrix.ignore_opts }} - test_filter: ${{ matrix.test_filter }} - setup_agentic_deps: ${{ matrix.setup_agentic_deps }} - upload_benchmarks: ${{ matrix.upload_benchmarks }} - genai_bench_image: ${{ env.GENAI_BENCH_IMAGE }} - hf_token: ${{ secrets.HF_TOKEN }} - openai_api_key: ${{ secrets.OPENAI_API_KEY }} - xai_api_key: ${{ secrets.XAI_API_KEY }} - anthropic_api_key: ${{ secrets.CLAUDE_API_KEY }} + - name: Checkout code + uses: actions/checkout@v6 + + - name: Check shared services + if: matrix.setup_agentic_deps == 'true' + run: bash scripts/ci_agentic_svc_deps.sh check --oracle oracle-db --brave brave-search-mcp + + - name: Setup vLLM backend + if: matrix.setup_vllm == 'true' + uses: ./.github/actions/setup-vllm + + - name: Setup TRT-LLM backend + if: matrix.setup_trtllm == 'true' + uses: ./.github/actions/setup-trtllm + + - name: Setup SGLang backend + if: matrix.setup_vllm != 'true' && matrix.setup_trtllm != 'true' + uses: ./.github/actions/setup-sglang + + - name: Download wheel artifact + uses: actions/download-artifact@v8 + with: + name: smg-wheel + path: wheel/ + + - name: Download WASM test fixtures + uses: actions/download-artifact@v8 + with: + name: wasm-test-fixtures + path: wasm/tests/fixtures/ + continue-on-error: true + + - name: Download Python client types + uses: actions/download-artifact@v8 + with: + name: python-client-types + path: clients/python/smg_client/types/ + + - name: Install wheel and test dependencies + run: | + pip uninstall -y smg || true + pip install wheel/*.whl + bash scripts/ci_install_e2e_deps.sh ${{ matrix.extra_deps }} + + - name: Pull genai-bench image + if: matrix.upload_benchmarks == 'true' + run: docker pull ${{ env.GENAI_BENCH_IMAGE }} + + - name: Setup Oracle + if: matrix.setup_agentic_deps == 'true' + run: | + bash scripts/ci_agentic_svc_deps.sh setup-oracle-client + bash scripts/ci_agentic_svc_deps.sh create-oracle-user oracle-db + bash scripts/ci_agentic_svc_deps.sh create-oracle-flyway-user oracle-db + + - name: Run E2E tests + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_API_KEY }} + BRAVE_MCP_HOST: ${{ matrix.setup_agentic_deps == 'true' && 'brave-search-mcp' || '' }} + run: | + bash scripts/ci_killall_sglang.sh "nuk_gpus" + ${{ matrix.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.ignore_opts }} ${{ matrix.test_dirs }} ${{ matrix.test_filter }} -s -vv -o log_cli=true --log-cli-level=INFO + + - name: Cleanup Oracle test users + if: always() && matrix.setup_agentic_deps == 'true' + run: | + bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-flyway-user oracle-db + bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-user oracle-db + + - name: Upload benchmark results + if: matrix.upload_benchmarks == 'true' + uses: actions/upload-artifact@v7 + with: + name: genai-bench-results-all-policies + path: benchmark_**/ e2e-chat-completions: name: ${{ matrix.name }} @@ -457,22 +516,55 @@ jobs: runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} steps: - - uses: actions/checkout@v6 - - uses: ./.github/actions/run-e2e - with: - setup_vllm: ${{ matrix.setup_vllm }} - setup_trtllm: ${{ matrix.setup_trtllm }} - test_dirs: ${{ matrix.test_dirs }} - extra_deps: ${{ matrix.extra_deps }} - env_vars: ${{ matrix.env_vars }} - reruns: ${{ matrix.reruns }} - parallel_opts: ${{ matrix.parallel_opts }} - ignore_opts: ${{ matrix.ignore_opts }} - test_filter: ${{ matrix.test_filter }} - hf_token: ${{ secrets.HF_TOKEN }} - openai_api_key: ${{ secrets.OPENAI_API_KEY }} - xai_api_key: ${{ secrets.XAI_API_KEY }} - anthropic_api_key: ${{ secrets.CLAUDE_API_KEY }} + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup vLLM backend + if: matrix.setup_vllm == 'true' + uses: ./.github/actions/setup-vllm + + - name: Setup TRT-LLM backend + if: matrix.setup_trtllm == 'true' + uses: ./.github/actions/setup-trtllm + + - name: Setup SGLang backend + if: matrix.setup_vllm != 'true' && matrix.setup_trtllm != 'true' + uses: ./.github/actions/setup-sglang + + - name: Download wheel artifact + uses: actions/download-artifact@v8 + with: + name: smg-wheel + path: wheel/ + + - name: Download WASM test fixtures + uses: actions/download-artifact@v8 + with: + name: wasm-test-fixtures + path: wasm/tests/fixtures/ + continue-on-error: true + + - name: Download Python client types + uses: actions/download-artifact@v8 + with: + name: python-client-types + path: clients/python/smg_client/types/ + + - name: Install wheel and test dependencies + run: | + pip uninstall -y smg || true + pip install wheel/*.whl + bash scripts/ci_install_e2e_deps.sh ${{ matrix.extra_deps }} + + - name: Run E2E tests + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_API_KEY }} + run: | + bash scripts/ci_killall_sglang.sh "nuk_gpus" + ${{ matrix.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.ignore_opts }} ${{ matrix.test_dirs }} ${{ matrix.test_filter }} -s -vv -o log_cli=true --log-cli-level=INFO e2e-agentic: name: ${{ matrix.name }} @@ -505,21 +597,64 @@ jobs: runs-on: k8s-runner-gpu timeout-minutes: ${{ matrix.timeout }} steps: - - uses: actions/checkout@v6 - - uses: ./.github/actions/run-e2e - with: - test_dirs: ${{ matrix.test_dirs }} - extra_deps: ${{ matrix.extra_deps }} - env_vars: ${{ matrix.env_vars }} - reruns: ${{ matrix.reruns }} - parallel_opts: ${{ matrix.parallel_opts }} - ignore_opts: ${{ matrix.ignore_opts }} - test_filter: ${{ matrix.test_filter }} - setup_agentic_deps: ${{ matrix.setup_agentic_deps }} - hf_token: ${{ secrets.HF_TOKEN }} - openai_api_key: ${{ secrets.OPENAI_API_KEY }} - xai_api_key: ${{ secrets.XAI_API_KEY }} - anthropic_api_key: ${{ secrets.CLAUDE_API_KEY }} + - name: Checkout code + uses: actions/checkout@v6 + + - name: Check shared services + if: matrix.setup_agentic_deps == 'true' + run: bash scripts/ci_agentic_svc_deps.sh check --oracle oracle-db --brave brave-search-mcp + + - name: Setup SGLang backend + uses: ./.github/actions/setup-sglang + + - name: Download wheel artifact + uses: actions/download-artifact@v8 + with: + name: smg-wheel + path: wheel/ + + - name: Download WASM test fixtures + uses: actions/download-artifact@v8 + with: + name: wasm-test-fixtures + path: wasm/tests/fixtures/ + continue-on-error: true + + - name: Download Python client types + uses: actions/download-artifact@v8 + with: + name: python-client-types + path: clients/python/smg_client/types/ + + - name: Install wheel and test dependencies + run: | + pip uninstall -y smg || true + pip install wheel/*.whl + bash scripts/ci_install_e2e_deps.sh ${{ matrix.extra_deps }} + + - name: Setup Oracle + if: matrix.setup_agentic_deps == 'true' + run: | + bash scripts/ci_agentic_svc_deps.sh setup-oracle-client + bash scripts/ci_agentic_svc_deps.sh create-oracle-user oracle-db + bash scripts/ci_agentic_svc_deps.sh create-oracle-flyway-user oracle-db + + - name: Run E2E tests + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_API_KEY }} + BRAVE_MCP_HOST: ${{ matrix.setup_agentic_deps == 'true' && 'brave-search-mcp' || '' }} + run: | + bash scripts/ci_killall_sglang.sh "nuk_gpus" + ${{ matrix.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.ignore_opts }} ${{ matrix.test_dirs }} ${{ matrix.test_filter }} -s -vv -o log_cli=true --log-cli-level=INFO + + - name: Cleanup Oracle test users + if: always() && matrix.setup_agentic_deps == 'true' + run: | + bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-flyway-user oracle-db + bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-user oracle-db go-unit-tests: name: go-unit-tests From 3980566928c5382e5b17b0da589f8e1431e4cc0b Mon Sep 17 00:00:00 2001 From: Chang Su Date: Wed, 4 Mar 2026 14:04:50 -0800 Subject: [PATCH 12/12] ci: revert matrix booleans and step conditions to match main Signed-off-by: Chang Su --- .github/workflows/pr-test-rust.yml | 64 ++++++++---------------------- 1 file changed, 17 insertions(+), 47 deletions(-) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index a141754d90..3e8a20bc43 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -346,10 +346,7 @@ jobs: extra_deps: "" env_vars: "" reruns: "" - setup_vllm: 'false' - setup_trtllm: 'false' - setup_agentic_deps: 'false' - upload_benchmarks: 'true' + upload_benchmarks: true parallel_opts: "" # No parallel for benchmarks (performance measurement) ignore_opts: "--ignore=e2e_test/benchmarks/test_go_bindings_perf.py --ignore=e2e_test/benchmarks/test_nightly_perf.py" # Go and nightly benchmarks run in dedicated jobs runner: 4-gpu-h100 @@ -360,10 +357,6 @@ jobs: extra_deps: "pytest-parallel py sentence-transformers" env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" reruns: "--reruns 2 --reruns-delay 5" - setup_vllm: 'false' - setup_trtllm: 'false' - setup_agentic_deps: 'false' - upload_benchmarks: 'false' parallel_opts: "--workers 1 --tests-per-worker 4" # Thread-based parallelism ignore_opts: "" runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} @@ -372,20 +365,16 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - - name: Check shared services - if: matrix.setup_agentic_deps == 'true' - run: bash scripts/ci_agentic_svc_deps.sh check --oracle oracle-db --brave brave-search-mcp - - name: Setup vLLM backend - if: matrix.setup_vllm == 'true' + if: matrix.setup_vllm uses: ./.github/actions/setup-vllm - name: Setup TRT-LLM backend - if: matrix.setup_trtllm == 'true' + if: matrix.setup_trtllm uses: ./.github/actions/setup-trtllm - name: Setup SGLang backend - if: matrix.setup_vllm != 'true' && matrix.setup_trtllm != 'true' + if: ${{ !matrix.setup_vllm && !matrix.setup_trtllm }} uses: ./.github/actions/setup-sglang - name: Download wheel artifact @@ -414,35 +403,21 @@ jobs: bash scripts/ci_install_e2e_deps.sh ${{ matrix.extra_deps }} - name: Pull genai-bench image - if: matrix.upload_benchmarks == 'true' + if: matrix.upload_benchmarks run: docker pull ${{ env.GENAI_BENCH_IMAGE }} - - name: Setup Oracle - if: matrix.setup_agentic_deps == 'true' - run: | - bash scripts/ci_agentic_svc_deps.sh setup-oracle-client - bash scripts/ci_agentic_svc_deps.sh create-oracle-user oracle-db - bash scripts/ci_agentic_svc_deps.sh create-oracle-flyway-user oracle-db - - name: Run E2E tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} XAI_API_KEY: ${{ secrets.XAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_API_KEY }} - BRAVE_MCP_HOST: ${{ matrix.setup_agentic_deps == 'true' && 'brave-search-mcp' || '' }} run: | bash scripts/ci_killall_sglang.sh "nuk_gpus" ${{ matrix.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.ignore_opts }} ${{ matrix.test_dirs }} ${{ matrix.test_filter }} -s -vv -o log_cli=true --log-cli-level=INFO - - name: Cleanup Oracle test users - if: always() && matrix.setup_agentic_deps == 'true' - run: | - bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-flyway-user oracle-db - bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-user oracle-db - - name: Upload benchmark results - if: matrix.upload_benchmarks == 'true' + if: matrix.upload_benchmarks && success() uses: actions/upload-artifact@v7 with: name: genai-bench-results-all-policies @@ -474,8 +449,6 @@ jobs: reruns: "--reruns 2 --reruns-delay 5" parallel_opts: "--workers 1 --tests-per-worker 4" test_filter: "" - setup_vllm: 'false' - setup_trtllm: 'false' ignore_opts: "" - name: chat-completions-vllm timeout: 20 @@ -487,8 +460,7 @@ jobs: # TODO: Remove filter when vLLM supports logprobs and n>1 with greedy sampling # Excludes: 5-grpc (logprobs=5), 2-None-grpc (n=2 with no logprobs), multiple_choices (n=2 tests) test_filter: "-k 'not (5-grpc or 2-None-grpc or multiple_choices)'" - setup_vllm: 'true' - setup_trtllm: 'false' + setup_vllm: true ignore_opts: "" - name: chat-completions-trtllm timeout: 60 @@ -498,8 +470,7 @@ jobs: reruns: "--reruns 2 --reruns-delay 5" parallel_opts: "--workers 1 --tests-per-worker 4" test_filter: "" - setup_vllm: 'false' - setup_trtllm: 'true' + setup_trtllm: true ignore_opts: "" runner: 4-gpu-h100 - name: vllm-pd @@ -510,8 +481,7 @@ jobs: reruns: "--reruns 2 --reruns-delay 5" parallel_opts: "" test_filter: "" - setup_vllm: 'true' - setup_trtllm: 'false' + setup_vllm: true ignore_opts: "" runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} @@ -520,15 +490,15 @@ jobs: uses: actions/checkout@v6 - name: Setup vLLM backend - if: matrix.setup_vllm == 'true' + if: matrix.setup_vllm uses: ./.github/actions/setup-vllm - name: Setup TRT-LLM backend - if: matrix.setup_trtllm == 'true' + if: matrix.setup_trtllm uses: ./.github/actions/setup-trtllm - name: Setup SGLang backend - if: matrix.setup_vllm != 'true' && matrix.setup_trtllm != 'true' + if: ${{ !matrix.setup_vllm && !matrix.setup_trtllm }} uses: ./.github/actions/setup-sglang - name: Download wheel artifact @@ -590,7 +560,7 @@ jobs: extra_deps: "" env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1" reruns: "--reruns 2 --reruns-delay 5" - setup_agentic_deps: 'true' + setup_agentic_deps: true parallel_opts: "" ignore_opts: "" test_filter: "-m 'not external'" @@ -601,7 +571,7 @@ jobs: uses: actions/checkout@v6 - name: Check shared services - if: matrix.setup_agentic_deps == 'true' + if: matrix.setup_agentic_deps run: bash scripts/ci_agentic_svc_deps.sh check --oracle oracle-db --brave brave-search-mcp - name: Setup SGLang backend @@ -633,7 +603,7 @@ jobs: bash scripts/ci_install_e2e_deps.sh ${{ matrix.extra_deps }} - name: Setup Oracle - if: matrix.setup_agentic_deps == 'true' + if: matrix.setup_agentic_deps run: | bash scripts/ci_agentic_svc_deps.sh setup-oracle-client bash scripts/ci_agentic_svc_deps.sh create-oracle-user oracle-db @@ -645,13 +615,13 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} XAI_API_KEY: ${{ secrets.XAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_API_KEY }} - BRAVE_MCP_HOST: ${{ matrix.setup_agentic_deps == 'true' && 'brave-search-mcp' || '' }} + BRAVE_MCP_HOST: ${{ matrix.setup_agentic_deps && 'brave-search-mcp' || '' }} run: | bash scripts/ci_killall_sglang.sh "nuk_gpus" ${{ matrix.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.ignore_opts }} ${{ matrix.test_dirs }} ${{ matrix.test_filter }} -s -vv -o log_cli=true --log-cli-level=INFO - name: Cleanup Oracle test users - if: always() && matrix.setup_agentic_deps == 'true' + if: always() && matrix.setup_agentic_deps run: | bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-flyway-user oracle-db bash scripts/ci_agentic_svc_deps.sh cleanup-oracle-user oracle-db