ci: Split L5 GPU workflows from L4

.github/workflows/gpu_l4_golden_parity.yml

@@ -0,0 +1,97 @@
+name: "L4: Golden Checkpoint Parity (GPU)"
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - 'testdata/**'
+      - 'scripts/**'
+  schedule:
+    # Run weekly on Monday at 4am UTC (after golden regen on Sunday)
+    - cron: '0 4 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: gpu-l4-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  golden-tests:
+    name: L4 - Verify ONNX output matches HuggingFace golden checkpoints
+    runs-on:
+      - self-hosted
+      - "1ES.Pool=onnxruntime-ep-mobius-github-linux-a10"
+      - "JobId=gpu-golden-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Cache pip packages
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/pip
+          key: pip-gpu-${{ hashFiles('pyproject.toml', 'requirements/ci/requirements.txt') }}
+          restore-keys: |
+            pip-gpu-
+
+      - name: Cache HuggingFace models
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/huggingface
+          key: hf-gpu-${{ hashFiles('testdata/cases/**/*.yaml') }}
+          restore-keys: |
+            hf-gpu-
+
+      - name: Install PyTorch (CUDA)
+        run: pip install torch --index-url https://download.pytorch.org/whl/cu124
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements/ci/requirements.txt
+          pip install -e '.[testing,transformers]'
+
+      - name: Run L4 checkpoint-verified tests
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          pytest tests/e2e_golden_test.py \
+            -m golden \
+            -v \
+            --timeout=300 \
+            --junitxml=junit-l4.xml \
+            --cov=src --cov-report=xml --cov-branch \
+            --tb=short
+        timeout-minutes: 30
+
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v6
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: gpu-l4
+      - name: Upload L4 test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/codecov-action@v6
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: gpu-l4
+          report-type: test_results
+          files: junit-l4.xml
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: gpu-l4-test-results
+          path: junit-l4.xml
+          retention-days: 30

.github/workflows/gpu_l5_generation_e2e.yml

@@ -0,0 +1,97 @@
+name: "L5: End-to-End Generation (GPU)"
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - 'testdata/**'
+      - 'scripts/**'
+  schedule:
+    # Run weekly on Monday at 5am UTC (after L4 golden tests)
+    - cron: '0 5 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: gpu-l5-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  generation-tests:
+    name: L5 - Full token generation and decoding correctness
+    runs-on:
+      - self-hosted
+      - "1ES.Pool=onnxruntime-ep-mobius-github-linux-a10"
+      - "JobId=gpu-l5-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Cache pip packages
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/pip
+          key: pip-gpu-${{ hashFiles('pyproject.toml', 'requirements/ci/requirements.txt') }}
+          restore-keys: |
+            pip-gpu-
+
+      - name: Cache HuggingFace models
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/huggingface
+          key: hf-gpu-${{ hashFiles('testdata/cases/**/*.yaml') }}
+          restore-keys: |
+            hf-gpu-
+
+      - name: Install PyTorch (CUDA)
+        run: pip install torch --index-url https://download.pytorch.org/whl/cu124
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements/ci/requirements.txt
+          pip install -e '.[testing,transformers]'
+
+      - name: Run L5 generation E2E tests
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          pytest tests/e2e_golden_test.py \
+            -m generation \
+            -v \
+            --timeout=300 \
+            --junitxml=junit-l5.xml \
+            --cov=src --cov-report=xml --cov-branch \
+            --tb=short
+        timeout-minutes: 30
+
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v6
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: gpu-l5
+      - name: Upload L5 test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/codecov-action@v6
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: gpu-l5
+          report-type: test_results
+          files: junit-l5.xml
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: gpu-l5-test-results
+          path: junit-l5.xml
+          retention-days: 30

.github/workflows/gpu_tests.yml

@@ -1,4 +1,4 @@
-name: GPU Tests (L4/L5)
+name: "L4: Golden Checkpoint Parity (GPU)"
 
 on:
   push:
@@ -17,17 +17,17 @@ permissions:
   contents: read
 
 concurrency:
-  group: gpu-tests-${{ github.ref }}
+  group: gpu-l4-tests-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   golden-tests:
-    name: L4/L5 Golden Tests
+    name: L4 - Verify ONNX output matches HuggingFace golden checkpoints
     runs-on:
       - self-hosted
       - "1ES.Pool=onnxruntime-ep-mobius-github-linux-a10"
       - "JobId=gpu-golden-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"
-    timeout-minutes: 60
+    timeout-minutes: 45
     steps:
       - uses: actions/checkout@v6
 
@@ -73,25 +73,12 @@ jobs:
             --tb=short
         timeout-minutes: 30
 
-      - name: Run L5 generation E2E tests
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: |
-          pytest tests/e2e_golden_test.py \
-            -m generation \
-            -v \
-            --timeout=300 \
-            --junitxml=junit-l5.xml \
-            --cov=src --cov-report=xml --cov-append --cov-branch \
-            --tb=short
-        timeout-minutes: 30
-
       - name: Upload coverage to Codecov
         if: always()
         uses: codecov/codecov-action@v6
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          flags: gpu
+          flags: gpu-l4
       - name: Upload L4 test results to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v6
@@ -100,21 +87,11 @@ jobs:
           flags: gpu-l4
           report-type: test_results
           files: junit-l4.xml
-      - name: Upload L5 test results to Codecov
-        if: ${{ !cancelled() }}
-        uses: codecov/codecov-action@v6
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          flags: gpu-l5
-          report-type: test_results
-          files: junit-l5.xml
 
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: gpu-test-results
-          path: |
-            junit-l4.xml
-            junit-l5.xml
+          name: gpu-l4-test-results
+          path: junit-l4.xml
           retention-days: 30