Adding evals after throughput benchmarks

.github/workflows/benchmark-tmpl.yml

@@ -45,7 +45,10 @@ on:
         required: false
         type: string
         default: '0.8'
-
+      run-eval:
+        type: boolean
+        required: false
+        default: false
 env:
   HF_TOKEN: ${{ secrets.HF_TOKEN }}
   HF_HUB_CACHE: '/mnt/hf_hub_cache/'
@@ -62,6 +65,7 @@ env:
   EP_SIZE: ${{ inputs.ep }}
   DP_ATTENTION: ${{ inputs.dp-attn }}
   CONC: ${{ inputs.conc }}
+  RUN_EVAL: ${{ inputs.run-eval }}
 
 permissions:
   contents: read
@@ -70,7 +74,7 @@ jobs:
   benchmark:
     runs-on: ${{ inputs.runner }}
     timeout-minutes: 180
-    name: '${{ inputs.exp-name }} ${{ inputs.runner }} ${{ inputs.framework }} ${{ inputs.precision }} tp=${{ inputs.tp }} ep=${{ inputs.ep }} dpa=${{ inputs.dp-attn }} conc=${{ inputs.conc }}'
+    name: '${{ inputs.exp-name }} ${{ inputs.runner }} ${{ inputs.framework }} ${{ inputs.precision }} ${{ inputs.run-eval && ''eval '' || '''' }}tp=${{ inputs.tp }} ep=${{ inputs.ep }} dpa=${{ inputs.dp-attn }} conc=${{ inputs.conc }}'
     steps:
       - name: Resource cleanup
         run: |
@@ -131,6 +135,9 @@ jobs:
         env:
           RUNNER_NAME: ${{ runner.name }}
           RESULT_FILENAME: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}_ep${{ env.EP_SIZE }}_dpa_${{ env.DP_ATTENTION }}_conc${{ env.CONC }}_${{ runner.name }}
+          # Suppress per-job eval markdown from being appended to the step summary.
+          # We'll publish a single combined eval table in the collection job instead.
+          GITHUB_STEP_SUMMARY: ''
         run: |
           bash ./runners/launch_${RUNNER_NAME%%_*}.sh
           FOUND_RESULT_FILE=
@@ -158,4 +165,21 @@ jobs:
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         with:
           name: ${{ env.RESULT_FILENAME }}
-          path: agg_${{ env.RESULT_FILENAME }}.json
+          path: agg_${{ env.RESULT_FILENAME }}.json
+
+      - name: Upload eval results (if any)
+        if: ${{ env.RUN_EVAL == 'true' }}
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: eval_${{ env.EXP_NAME }}_${{ env.RESULT_FILENAME }}
+          path: |
+            meta_env.json
+            results*.json
+          if-no-files-found: ignore
+
+      - name: Cleanup eval outputs (post-upload)
+        if: ${{ env.RUN_EVAL == 'true' }}
+        run: |
+          rm -f meta_env.json || true
+          # Remove any eval results JSONs that were moved into workspace
+          rm -f results*.json || true

.github/workflows/collect-evals.yml

@@ -0,0 +1,45 @@
+name: Template - Collect Evals
+
+on:
+  workflow_call:
+    inputs:
+      exp-name:
+        required: false
+        type: string
+        default: ''
+
+permissions:
+  contents: read
+
+jobs:
+  collect-evals:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          token: ${{ secrets.REPO_PAT }}
+          fetch-depth: 0
+
+      - name: Download eval artifacts
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+        with:
+          path: eval_results/
+          pattern: ${{ inputs.exp-name && format('eval_{0}_*', inputs.exp-name) || 'eval_*' }}
+
+      - name: Summarize evals
+        run: |
+          echo "## Eval Summary - ${{ inputs.exp-name || 'all' }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          python3 utils/collect_eval_results.py eval_results/ ${{ inputs.exp-name || 'all' }} >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload aggregated evals
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: eval_results_${{ inputs.exp-name || 'all' }}
+          path: agg_eval_${{ inputs.exp-name || 'all' }}.json
+
+      - name: Cleanup downloaded eval artifacts
+        if: ${{ always() }}
+        run: |
+          rm -rf eval_results/ || true

.github/workflows/eval-gms8k.yml

@@ -0,0 +1,64 @@
+name: Eval - GSM8K (PoC)
+
+on:
+  workflow_dispatch:
+    inputs:
+      exp-name:
+        description: "Experiment name (prefix selects docker script)"
+        required: false
+        type: string
+        default: "gptoss_gsm8k_poc"
+      image:
+        description: "Serving image"
+        required: false
+        type: string
+        default: "vllm/vllm-openai:v0.11.0"
+      model:
+        description: "Model"
+        required: false
+        type: string
+        default: "openai/gpt-oss-120b"
+      tp:
+        description: "Tensor Parallel Size"
+        required: false
+        type: string
+        default: "2"
+      port:
+        description: "Server port"
+        required: false
+        type: string
+        default: "8888"
+      num_fewshot:
+        description: "Fewshot k for GSM8K"
+        required: false
+        type: string
+        default: "5"
+      limit:
+        description: "Sample limit for GSM8K"
+        required: false
+        type: string
+        default: "1300"
+  push:
+    paths:
+      - '.github/workflows/eval-gms8k.yml'
+      - '.github/workflows/eval-tmpl.yml' 
+      - 'benchmarks/benchmark_lib.sh'
+
+jobs:
+  eval:
+    uses: ./.github/workflows/eval-tmpl.yml
+    secrets: inherit
+    with:
+      runner: h100-cw_0
+      image: ${{ inputs.image || 'vllm/vllm-openai:v0.11.0' }}
+      model: ${{ inputs.model || 'openai/gpt-oss-120b' }}
+      framework: vllm
+      precision: fp4
+      exp-name: ${{ inputs.exp-name || 'gptoss_gsm8k_poc' }}
+      tp: '4'
+      ep: '1'
+      dp-attn: false
+      port: ${{ inputs.port || '8888' }}
+      eval-task: gsm8k
+      num-fewshot: ${{ inputs.num_fewshot || '5' }}
+      limit: ${{ inputs.limit || '200' }} 

.github/workflows/eval-tmpl.yml

@@ -0,0 +1,152 @@
+name: Template - Eval
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        required: true
+        type: string
+      image:
+        required: true
+        type: string
+      model:
+        required: true
+        type: string
+      framework:
+        required: true
+        type: string
+      precision:
+        required: true
+        type: string
+      exp-name:
+        required: true
+        type: string
+      tp:
+        required: true
+        type: string
+      ep:
+        required: false
+        type: string
+        default: '1'
+      dp-attn:
+        required: false
+        type: boolean
+        default: false
+      port:
+        required: false
+        type: string
+        default: '8888'
+      eval-task:
+        required: true
+        type: string
+      num-fewshot:
+        required: false
+        type: string
+        default: '5'
+      limit:
+        required: false
+        type: string
+        default: '200'
+
+env:
+  HF_TOKEN: ${{ secrets.HF_TOKEN }}
+  HF_HUB_CACHE: '/mnt/hf_hub_cache/'
+  EXP_NAME: ${{ inputs.exp-name }}
+  MODEL: ${{ inputs.model }}
+  IMAGE: ${{ inputs.image }}
+  FRAMEWORK: ${{ inputs.framework }}
+  PRECISION: ${{ inputs.precision }}
+  TP: ${{ inputs.tp }}
+  EP_SIZE: ${{ inputs.ep }}
+  DP_ATTENTION: ${{ inputs.dp-attn }}
+  PORT: ${{ inputs.port }}
+  EVAL_TASK: ${{ inputs['eval-task'] }}
+  NUM_FEWSHOT: ${{ inputs['num-fewshot'] }}
+  LIMIT: ${{ inputs.limit }}
+  # Keep eval outputs only under /tmp
+  EVAL_RESULT_DIR: /tmp/eval_out
+  CONC: '32'
+  MAX_MODEL_LEN: '4096'
+  ISL: 1024
+  OSL: 1024
+  RANDOM_RANGE_RATIO: '0.8'
+  RESULT_FILENAME: results
+  RUN_EVAL: true
+
+jobs:
+  eval:
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 180
+    name: "Eval ${{ inputs.exp-name }} ${{ inputs.runner }} ${{ inputs.precision }} tp=${{ inputs.tp }} task=${{ inputs['eval-task'] }} limit=${{ inputs.limit }}"
+    steps:
+      - name: Resource cleanup
+        run: |
+          sudo rm -rf /home/nvadmin/actions-runner/_work/InferenceMAX/InferenceMAX/eval_out/
+          # Helper to avoid indefinite hangs on flaky tools (Docker/Slurm)
+          safe_timeout() {
+            if command -v timeout >/dev/null 2>&1; then
+              timeout -k 5 30s "$@"
+            else
+              "$@"
+            fi
+          }
+          host=$(hostname)
+          if [[ "$host" == "b200-81" || "$host" == "b200-80" || "$host" == "b200-79" ]]; then
+            if command -v docker >/dev/null 2>&1; then
+              echo "[INFO] Running container-by-container cleanup on $host"
+              cids=$(safe_timeout docker ps -aq || true)
+              for cid in $cids; do
+                echo "[INFO] Cleaning container $cid"
+                safe_timeout docker stop -t 90 "$cid" || true
+                safe_timeout docker wait "$cid" >/dev/null 2>&1 || true
+                safe_timeout docker rm -f "$cid" >/dev/null 2>&1 || true
+              done
+              sleep 2
+              if nvidia-smi --query-compute-apps=pid --format=csv,noheader | grep -q '[0-9]'; then
+                echo "[WARN] After stop, GPU still busy:"
+                nvidia-smi || true
+              fi
+            else
+              echo "[Docker] docker client not found; skipping cleanup"
+            fi
+          else
+            echo "[Docker] skipping docker cleanup on host $host"
+          fi
+          # Best-effort cleanup of prior eval outputs; do not block
+
+          if command -v squeue >/dev/null 2>&1; then
+            echo "[Slurm] Cleaning up resources ..."
+            safe_timeout scancel -u "$USER" || true
+            # Wait up to 5 minutes for jobs to clear to avoid indefinite hang
+            end=$((SECONDS + 300))
+            while [ $SECONDS -lt $end ]; do
+              queued=$(safe_timeout squeue -u "$USER" --noheader --format='%i' 2>/dev/null || true)
+              if [ -z "$queued" ]; then
+                break
+              fi
+              echo "$queued" | sed 's/^/[Slurm] pending job: /' || true
+              sleep 5
+            done
+            # Final status; do not block
+            safe_timeout squeue -u "$USER" || true
+            if [ -n "$(safe_timeout squeue -u "$USER" --noheader --format='%i' 2>/dev/null || true)" ]; then
+              echo "[Slurm] Jobs still present after timeout; proceeding"
+            fi
+          fi
+
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          # Avoid aggressive workspace deletion if stale, rely on git reset/clean later
+          clean: true
+
+      - name: Launch eval via runner script
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+          RUN_MODE: eval
+          # Optional: structured filename if runner chooses to use it later
+          EVAL_RESULT_BASENAME: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_${{ runner.name }}
+        run: |
+          bash ./runners/launch_${RUNNER_NAME%%_*}.sh
+
+      # Intentionally no eval artifact uploads: eval outputs remain in /tmp only.

.github/workflows/full-sweep-1k1k-scheduler.yml

@@ -17,7 +17,7 @@ jobs:
             - id: get-dsr1-configs
               run: |
                   pip install pydantic
-                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix dsr1)
+                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix dsr1 --run-evals)
                   echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
 
     get-gptoss-configs:
@@ -31,7 +31,7 @@ jobs:
             - id: get-gptoss-configs
               run: |
                   pip install pydantic
-                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix gptoss)
+                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix gptoss --run-evals)
                   echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
 
     benchmark-dsr1:

.github/workflows/full-sweep-1k8k-scheduler.yml

@@ -17,7 +17,7 @@ jobs:
             - id: get-dsr1-configs
               run: |
                   pip install pydantic
-                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix dsr1)
+                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix dsr1 --run-evals)
                   echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
 
     get-gptoss-configs:
@@ -31,7 +31,7 @@ jobs:
             - id: get-gptoss-configs
               run: |
                   pip install pydantic
-                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix gptoss)
+                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix gptoss --run-evals)
                   echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
 
     benchmark-dsr1:

.github/workflows/full-sweep-8k1k-scheduler.yml

@@ -17,7 +17,7 @@ jobs:
             - id: get-dsr1-configs
               run: |
                   pip install pydantic
-                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 8k1k --model-prefix dsr1)
+                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 8k1k --model-prefix dsr1 --run-evals)
                   echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
 
     get-gptoss-configs:
@@ -31,7 +31,7 @@ jobs:
             - id: get-gptoss-configs
               run: |
                   pip install pydantic
-                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 8k1k --model-prefix gptoss)
+                  CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 8k1k --model-prefix gptoss --run-evals)
                   echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
 
     benchmark-dsr1: