From c4b136bcc86279915b090b48ba1a6b90c34911e4 Mon Sep 17 00:00:00 2001 From: Saravana Periyasamy Date: Fri, 2 Oct 2026 11:29:11 -0500 Subject: [PATCH 1/4] ci(recipes): cap DeepSeek aggregated perf concurrency at 128 Signed-off-by: Saravana Periyasamy --- .github/ci/deepseek-v4-pro-agg/patch-perf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml index 6b85afc54dd0..06ce3e82a6a7 100644 --- a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml +++ b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml @@ -24,7 +24,7 @@ spec: - name: ENDPOINT value: dsv4-pro-agg-frontend:8000 - name: CONCURRENCIES - value: "1 8 64 128 256 512 1024" + value: "1 8 64 128" volumes: - name: model-cache persistentVolumeClaim: From 93b208e85dad8539e0737ef7eabc01deb88ff29d Mon Sep 17 00:00:00 2001 From: Saravana Periyasamy Date: Fri, 2 Oct 2026 11:39:41 -0500 Subject: [PATCH 2/4] ci(recipes): use one DeepSeek aggregated perf concurrency Signed-off-by: Saravana Periyasamy --- .github/ci/deepseek-v4-pro-agg/patch-perf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml index 06ce3e82a6a7..0a00e103dd2e 100644 --- a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml +++ b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml @@ -24,7 +24,7 @@ spec: - name: ENDPOINT value: dsv4-pro-agg-frontend:8000 - name: CONCURRENCIES - value: "1 8 64 128" + value: "128" volumes: - name: model-cache persistentVolumeClaim: From 188749bc448b8994455dfd88b42ce14d036b444c Mon Sep 17 00:00:00 2001 From: Saravana Periyasamy Date: Fri, 2 Oct 2026 12:05:00 -0500 Subject: [PATCH 3/4] ci(recipes): align Kimi aggregated nightly concurrency Signed-off-by: Saravana Periyasamy --- .github/ci/kimi-k25-agg/patch-perf.yaml | 4 ++-- .github/workflows/shared-recipe-perf.yml | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/ci/kimi-k25-agg/patch-perf.yaml b/.github/ci/kimi-k25-agg/patch-perf.yaml index 0ba0bd2c3009..0fd11e42a2aa 100644 --- a/.github/ci/kimi-k25-agg/patch-perf.yaml +++ b/.github/ci/kimi-k25-agg/patch-perf.yaml @@ -21,9 +21,9 @@ spec: - name: perf env: - name: CONCURRENCIES - value: "1" + value: "128" - name: BENCHMARK_DURATION - value: "60" + value: "120" volumes: - name: model-cache persistentVolumeClaim: diff --git a/.github/workflows/shared-recipe-perf.yml b/.github/workflows/shared-recipe-perf.yml index dd68d7904bec..57541b88bd98 100644 --- a/.github/workflows/shared-recipe-perf.yml +++ b/.github/workflows/shared-recipe-perf.yml @@ -257,15 +257,16 @@ jobs: if: ${{ success() && !inputs.preflight_only }} timeout-minutes: 5 run: | + concurrencies=$(yq -er '.spec.template.spec.containers[] | select(.name == strenv(PERF_CONTAINER)).env[] | select(.name == "CONCURRENCIES").value' "$ARTIFACTS/perf.yaml") if [[ "$RECIPE" == kimi-k25-agg ]]; then + [[ "$concurrencies" =~ ^[1-9][0-9]*$ ]] shopt -s nullglob - profiles=("$ARTIFACTS"/perf/*/*_trace_c1_*/profile_export_aiperf.json) + profiles=("$ARTIFACTS"/perf/*/*_trace_c"$concurrencies"_*/profile_export_aiperf.json) (( ${#profiles[@]} == 1 )) jq -e '.request_count.avg > 0 and (.error_request_count.avg // 0) == 0 and .output_sequence_length.avg > 0 and .was_cancelled != true' "${profiles[0]}" exit 0 fi - concurrencies=$(yq -er '.spec.template.spec.containers[] | select(.name == strenv(PERF_CONTAINER)).env[] | select(.name == "CONCURRENCIES").value' "$ARTIFACTS/perf.yaml") for concurrency in $concurrencies; do jq -e --argjson requests "$((concurrency * 3))" ' .request_count.avg == $requests and (.error_request_count.avg // 0) == 0 and From da2797f5ee301c371c7f9ec22937761d38b43b55 Mon Sep 17 00:00:00 2001 From: Saravana Periyasamy Date: Sun, 4 Oct 2026 01:57:05 -0500 Subject: [PATCH 4/4] ci(kimi): match nightly concurrency to trace sessions Signed-off-by: Saravana Periyasamy --- .github/ci/kimi-k25-agg/patch-perf.yaml | 2 +- .github/workflows/shared-recipe-perf.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ci/kimi-k25-agg/patch-perf.yaml b/.github/ci/kimi-k25-agg/patch-perf.yaml index 0fd11e42a2aa..b0edc796ad6a 100644 --- a/.github/ci/kimi-k25-agg/patch-perf.yaml +++ b/.github/ci/kimi-k25-agg/patch-perf.yaml @@ -21,7 +21,7 @@ spec: - name: perf env: - name: CONCURRENCIES - value: "128" + value: "20" - name: BENCHMARK_DURATION value: "120" volumes: diff --git a/.github/workflows/shared-recipe-perf.yml b/.github/workflows/shared-recipe-perf.yml index 57541b88bd98..a1551ead463b 100644 --- a/.github/workflows/shared-recipe-perf.yml +++ b/.github/workflows/shared-recipe-perf.yml @@ -263,7 +263,8 @@ jobs: shopt -s nullglob profiles=("$ARTIFACTS"/perf/*/*_trace_c"$concurrencies"_*/profile_export_aiperf.json) (( ${#profiles[@]} == 1 )) - jq -e '.request_count.avg > 0 and (.error_request_count.avg // 0) == 0 and + # The CI trace has 20 conversations with 10 turns each. + jq -e '.request_count.avg == 200 and (.error_request_count.avg // 0) == 0 and .output_sequence_length.avg > 0 and .was_cancelled != true' "${profiles[0]}" exit 0 fi