diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 4041f31bc3..8667e762c7 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -25,9 +25,14 @@ on: inputs: test_to_run: required: false - default: all - type: string - description: Comma-separated list of tests to run. Use "all" to run the full test suite. + default: L2 + type: choice + options: + - docs + - L0 + - L1 + - L2 + description: Test level to run. docs = doc tests only, L0 = unit/docs/lint, L1 = L0 + functional, L2 = L1 + convergence # TODO: Due to limited compute, disabling pushes to main. This is okay to do since we force PRs to be up to date and the CI tests on pull/$PR_NUM/merge #push: # branches: @@ -41,20 +46,8 @@ jobs: pre-flight: runs-on: ubuntu-latest outputs: - test_to_run: ${{ steps.test_to_run.outputs.main }} - all: ${{ steps.all.outputs.main }} - run_ci: ${{ steps.evaluate.outputs.run_ci }} + test_level: ${{ steps.evaluate.outputs.test_level }} steps: - - name: Parse test_to_run - id: test_to_run - run: | - parsed_string=$(echo ${{ inputs.test_to_run || 'all' }} | jq -c --raw-input 'split(",")') - echo "main=${parsed_string}" | tee -a "$GITHUB_OUTPUT" - - name: Parse all - id: all - run: | - echo "main=${{ contains(fromJSON(steps.test_to_run.outputs.main), 'all') }}" | tee -a "$GITHUB_OUTPUT" - - name: Get changed files id: changed-files if: github.event_name == 'pull_request' @@ -81,19 +74,34 @@ jobs: # Some output that's helpful for debugging echo "Docs changed: $CHANGED_DOCS" echo "Src changed: $CHANGED_SRC" - - # echo "DOCS_ONLY: $DOCS_ONLY" echo "LABEL: $LABEL" echo "IS_PULLREQUEST: $IS_PULLREQUEST" - # Run CI only (on main or if label is attached) and if it's not only docs - echo run_ci=$([[ ("$LABEL" = "true" || "$IS_PULLREQUEST" = "false" || "$MERGE_GROUP" = "true") && "$DOCS_ONLY" = "false" ]] && echo "true" || echo "false") | tee -a "$GITHUB_OUTPUT" + # Determine test level based on conditions + if [[ "$DOCS_ONLY" == "true" ]]; then + # For doc-only changes, run only doc tests + TEST_LEVEL="docs" + elif [[ "$LABEL" == "true" || "$IS_PULLREQUEST" == "false" || "$MERGE_GROUP" == "true" ]]; then + # For labeled PRs, pushes to main (IS_PULL_REQUEST=false), or merge group events, run L0 by default + TEST_LEVEL="L0" + else + # Skip tests by default for non-labeled PRs + TEST_LEVEL="none" + fi + + # Override test level if specified in workflow_dispatch + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "Overriding test level from $TEST_LEVEL to ${{ inputs.test_to_run }}" + TEST_LEVEL="${{ inputs.test_to_run }}" + fi + + echo "test_level=$TEST_LEVEL" | tee -a "$GITHUB_OUTPUT" lint-check: name: Lint check needs: [pre-flight] runs-on: ubuntu-latest - if: ${{ needs.pre-flight.outputs.run_ci == 'true' }} + if: ${{ needs.pre-flight.outputs.test_level != 'none' }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -107,7 +115,7 @@ jobs: name: Sphinx build needs: [pre-flight] runs-on: ubuntu-latest - if: ${{ needs.pre-flight.outputs.run_ci == 'true' }} + if: ${{ needs.pre-flight.outputs.test_level != 'none' }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -118,7 +126,7 @@ jobs: uv run --group docs sphinx-build . _build/html build-container: - if: ${{ needs.pre-flight.outputs.run_ci == 'true' }} + if: ${{ needs.pre-flight.outputs.test_level != 'none' }} needs: [pre-flight] uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_container.yml@v0.22.7 with: @@ -134,29 +142,53 @@ jobs: name: Tests needs: [build-container, pre-flight] uses: ./.github/workflows/_run_test.yml - if: ${{ needs.pre-flight.outputs.run_ci == 'true' }} + if: ${{ needs.pre-flight.outputs.test_level != 'none' }} with: RUNNER: self-hosted-azure TIMEOUT: 15 UNIT_TEST_SCRIPT: | cd /opt/reinforcer - uv run --no-sync bash -x ./tests/run_unit.sh + if [[ "${{ needs.pre-flight.outputs.test_level }}" =~ ^(L0|L1|L2)$ ]]; then + uv run --no-sync bash -x ./tests/run_unit.sh + else + echo Skipping unit tests for docs-only level + fi DOC_TEST_SCRIPT: | cd /opt/reinforcer/docs - uv run --no-sync sphinx-build -b doctest . _build/doctest + if [[ "${{ needs.pre-flight.outputs.test_level }}" =~ ^(docs|L0|L1|L2)$ ]]; then + uv run --no-sync sphinx-build -b doctest . _build/doctest + else + echo Skipping doc tests for level ${{ needs.pre-flight.outputs.test_level }} + fi FUNCTIONAL_TEST_SCRIPT: | - # TODO: Temporarily disable functional tests until we have more capacity and tests run quicker - # Related: https://github.com/NVIDIA/reinforcer/pull/27 - # cd /opt/reinforcer - # uv run --no-sync bash ./tests/functional/sft.sh - # uv run --no-sync bash ./tests/functional/grpo.sh + cd /opt/reinforcer + if [[ "${{ needs.pre-flight.outputs.test_level }}" =~ ^(L1|L2)$ ]]; then + uv run --no-sync bash ./tests/functional/sft.sh + uv run --no-sync bash ./tests/functional/grpo.sh + else + echo Skipping functional tests for level ${{ needs.pre-flight.outputs.test_level }} + fi + # TODO: enable once we have convergence tests in CI + #CONVERGENCE_TEST_SCRIPT: | + # cd /opt/reinforcer + # if [[ "${{ needs.pre-flight.outputs.test_level }}" =~ ^(L2)$ ]]; then + # echo "Running convergence tests" + # # Add your convergence test commands here + # # uv run --no-sync bash ./tests/convergence/test.sh + # else + # echo "Skipping convergence tests for level ${{ needs.pre-flight.outputs.test_level }}" + # fi AFTER_SCRIPT: | cd /opt/reinforcer cat <- + ${{ + (needs.pre-flight.outputs.test_level == 'none') || + (needs.pre-flight.outputs.test_level != 'none' && + needs.lint-check.result == 'success' && + needs.sphinx-build.result == 'success' && + needs.tests.result == 'success') + }} CI_SKIP: ${{ github.event.label.name == 'Skip CICD' }} + TEST_LEVEL: ${{ needs.pre-flight.outputs.test_level }} run: | - SUMMARY=$(echo $JOB_RESULTS | jq 'to_entries[] | .key + ": " + .value.result' | tr -d '"') - echo '🤖: CICD Result' >> $GITHUB_STEP_SUMMARY + echo '🤖: CICD Result for test level: ${{ needs.pre-flight.outputs.test_level }}' >> $GITHUB_STEP_SUMMARY echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY + if [[ "$TEST_LEVEL" == "none" ]]; then + echo "No tests were run, passing gate" >> $GITHUB_STEP_SUMMARY + exit 0 + fi + test "$ALL_SUCCESS" = "true" || test "$CI_SKIP" = "true" DCO_merge_group: