[Feat] Success Rate Calculation

.github/workflows/full-sweep-scheduler.yml

@@ -23,3 +23,31 @@ jobs:
       use_mi300x: true
       use_mi325x: true
       use_mi355x: false
+
+  calc-success-rate:
+    needs: mega-run
+    runs-on: ubuntu-latest
+
+    env:
+      RESULTS_DIR: "results/"
+      STATS_FILENAME: "run_stats"
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.REPO_PAT }}
+          fetch-depth: 0
+
+      - name: Download results artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ env.RESULTS_DIR }}
+          pattern: results_*
+
+      - name: Calculate success rate
+        run: python3 utils/calc_success_rate.py $RESULTS_DIR $STATS_FILENAME
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: "run-stats"
+          path: ${{ env.STATS_FILENAME }}.json

.github/workflows/full-sweep-test.yml

@@ -50,3 +50,31 @@ jobs:
       use_mi300x: ${{ inputs.use_mi300x }}
       use_mi325x: ${{ inputs.use_mi325x }}
       use_mi355x: ${{ inputs.use_mi355x }}
+
+  calc-success-rate:
+    needs: mega-test-run
+    runs-on: ubuntu-latest
+
+    env:
+      RESULTS_DIR: "results/"
+      STATS_FILENAME: "run_stats"
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.REPO_PAT }}
+          fetch-depth: 0
+
+      - name: Download results artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ env.RESULTS_DIR }}
+          pattern: results_*
+
+      - name: Calculate success rate
+        run: python3 utils/calc_success_rate.py $RESULTS_DIR $STATS_FILENAME
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: "run-stats"
+          path: ${{ env.STATS_FILENAME }}.json

utils/calc_success_rate.py

@@ -0,0 +1,30 @@
+import sys
+import json
+from pathlib import Path
+
+n_iosl = 3   # [1k1k, 8k1k, 1k8k]
+n_concs = 5  # [4, 8, 16, 32, 64]
+total_runs = {
+    'h100': (3 + 4) * n_iosl * n_concs,                # 70b-tp: [2, 4, 8], gptoss-tp: [1, 2, 4, 8]
+    'h200': (4 + 1 + 4) * 2 * n_iosl * n_concs,        # (70b-tp: [1, 2, 4, 8], dsr1-tp: [8], gptoss-tp: [1, 2, 4, 8]) x [vllm/sglang, trt]
+    'b200': ((2 + 1) * 2 + 2) * 2 * n_iosl * n_concs,  # ((70b-tp: [1, 2], dsr1-tp: [8]) x [fp4, fp8], gptoss-tp: [1, 2]) x [vllm/sglang, trt]
+    'mi300x': (4 + 1 + 4) * n_iosl * n_concs,          # 70b-tp: [1, 2, 4, 8], dsr1-tp: [8], gptoss-tp: [1, 2, 4, 8]
+    'mi325x': (4 + 1 + 4) * n_iosl * n_concs,          # 70b-tp: [1, 2, 4, 8], dsr1-tp: [8], gptoss-tp: [1, 2, 4, 8]
+    'mi355x': ((2 + 1) * 2 + 2) * n_iosl * n_concs,    # (70b-tp: [1, 2], dsr1-tp: [8]) x [fp4, fp8], gptoss-tp: [1, 2]
+}
+success_runs = {'h100': 0, 'h200': 0, 'b200': 0, 'mi300x': 0, 'mi325x': 0, 'mi355x': 0}
+
+
+for results_filepath in Path(sys.argv[1]).rglob('*.json'):
+    with open(results_filepath) as f:
+        results = json.load(f)
+
+    for result in results:
+        hw_type = result['hw'].replace('-trt', '')
+        success_runs[hw_type] += 1
+
+run_stats = {}
+for hw, n_success in success_runs.items():
+    run_stats[hw] = {'n_success': n_success, 'total': total_runs[hw]}
+with open(f'{sys.argv[2]}.json', 'w') as f:
+    json.dump(run_stats, f, indent=2)