diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_l4_golden_parity.yml similarity index 70% rename from .github/workflows/gpu_tests.yml rename to .github/workflows/gpu_l4_golden_parity.yml index 86d68a8e..da88c20e 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_l4_golden_parity.yml @@ -1,4 +1,4 @@ -name: GPU Tests (L4/L5) +name: "L4: Golden Checkpoint Parity (GPU)" on: push: @@ -9,20 +9,20 @@ on: - 'testdata/**' - 'scripts/**' schedule: - # Run weekly on Monday at 4am UTC (after golden regen on Sunday) - - cron: '0 4 * * 1' + # Run nightly at 4am UTC (after golden regen on Sunday) + - cron: '0 4 * * *' workflow_dispatch: permissions: contents: read concurrency: - group: gpu-tests-${{ github.ref }} + group: gpu-l4-tests-${{ github.ref }} cancel-in-progress: true jobs: golden-tests: - name: L4/L5 Golden Tests + name: L4 - Verify ONNX output matches HuggingFace golden checkpoints runs-on: - self-hosted - "1ES.Pool=onnxruntime-ep-mobius-github-linux-a10" @@ -73,25 +73,12 @@ jobs: --tb=short timeout-minutes: 30 - - name: Run L5 generation E2E tests - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - pytest tests/e2e_golden_test.py \ - -m generation \ - -v \ - --timeout=300 \ - --junitxml=junit-l5.xml \ - --cov=src --cov-report=xml --cov-append --cov-branch \ - --tb=short - timeout-minutes: 30 - - name: Upload coverage to Codecov if: always() uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} - flags: gpu + flags: gpu-l4 - name: Upload L4 test results to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v6 @@ -100,21 +87,11 @@ jobs: flags: gpu-l4 report-type: test_results files: junit-l4.xml - - name: Upload L5 test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v6 - with: - token: ${{ secrets.CODECOV_TOKEN }} - flags: gpu-l5 - report-type: test_results - files: junit-l5.xml - name: Upload test results if: always() uses: actions/upload-artifact@v7 with: - name: gpu-test-results - path: | - junit-l4.xml - junit-l5.xml + name: gpu-l4-test-results + path: junit-l4.xml retention-days: 30 diff --git a/.github/workflows/gpu_l5_generation_e2e.yml b/.github/workflows/gpu_l5_generation_e2e.yml new file mode 100644 index 00000000..7b7b316b --- /dev/null +++ b/.github/workflows/gpu_l5_generation_e2e.yml @@ -0,0 +1,97 @@ +name: "L5: End-to-End Generation (GPU)" + +on: + push: + branches: [ "main" ] + paths: + - 'src/**' + - 'tests/**' + - 'testdata/**' + - 'scripts/**' + schedule: + # Run nightly at 5am UTC (after L4 golden tests) + - cron: '0 5 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: gpu-l5-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + generation-tests: + name: L5 - Full token generation and decoding correctness + runs-on: + - self-hosted + - "1ES.Pool=onnxruntime-ep-mobius-github-linux-a10" + - "JobId=gpu-l5-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Cache pip packages + uses: actions/cache@v5 + with: + path: ~/.cache/pip + key: pip-gpu-${{ hashFiles('pyproject.toml', 'requirements/ci/requirements.txt') }} + restore-keys: | + pip-gpu- + + - name: Cache HuggingFace models + uses: actions/cache@v5 + with: + path: ~/.cache/huggingface + key: hf-gpu-${{ hashFiles('testdata/cases/**/*.yaml') }} + restore-keys: | + hf-gpu- + + - name: Install PyTorch (CUDA) + run: pip install torch --index-url https://download.pytorch.org/whl/cu124 + + - name: Install dependencies + run: | + pip install -r requirements/ci/requirements.txt + pip install -e '.[testing,transformers]' + + - name: Run L5 generation E2E tests + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + pytest tests/e2e_golden_test.py \ + -m generation \ + -v \ + --timeout=300 \ + --junitxml=junit-l5.xml \ + --cov=src --cov-report=xml --cov-branch \ + --tb=short + timeout-minutes: 30 + + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v6 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: gpu-l5 + - name: Upload L5 test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v6 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: gpu-l5 + report-type: test_results + files: junit-l5.xml + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: gpu-l5-test-results + path: junit-l5.xml + retention-days: 30