Merge branch 'onur/inframework_deploy_test' of https://github.com/NVIDIA/NeMo into onur/inframework_deploy_test

Author: oyilmaz-nvidia

.coveragerc

@@ -1,5 +1,16 @@
+[run]
+concurrency = thread,multiprocessing
+omit =
+    /tmp/*
+    /home/TestData/*
+    /workspace/Megatron-LM/*
+    nemo/collections/nlp/*
+    nemo/collections/multimodal/*
+    nemo/collections/vision/*
+
 [paths]
 source =
     nemo/
     /home/runner/work/NeMo/NeMo/nemo
-    /workspace/nemo
+    /workspace/nemo
+

.github/workflows/_test_template.yml

@@ -50,6 +50,9 @@ jobs:
       potential_infra_failure: ${{ steps.check.outputs.potential_infra_failure }}
     env:
       DIR: ${{ github.run_id }}
+      IS_OPTIONAL: ${{ inputs.IS_OPTIONAL }}
+      SCRIPT: ${{ inputs.SCRIPT }}
+      IS_UNIT_TEST: ${{ inputs.IS_UNIT_TEST }}
     steps:
       - name: Docker system cleanup
         run: |
@@ -60,8 +63,6 @@ jobs:
           docker pull nemoci.azurecr.io/nemo_container:${{ github.run_id }}
 
       - name: Start container
-        env:
-          DIR: ${{ github.run_id }}
         run: |
           mkdir -p $DIR
 
@@ -90,10 +91,6 @@ jobs:
           bash $DIR/retry_job.sh
 
       - name: Create run-script
-        env:
-          DIR: ${{ github.run_id }}
-          SCRIPT: ${{ inputs.SCRIPT }}
-          IS_UNIT_TEST: ${{ inputs.IS_UNIT_TEST }}
         id: create
         run: |
 
@@ -110,7 +107,7 @@ jobs:
           (
             set -e
 
-            docker exec nemo_container_${{ github.run_id }} bash -c '$SCRIPT && echo "Finished successfully." || echo "Did not finish."'
+            docker exec -t nemo_container_${{ github.run_id }} bash -c 'RUN_ID=${{ github.run_id }} bash tests/functional_tests/$SCRIPT.sh && echo "Finished successfully." || echo "Did not finish."'
           ) 2>&1 | tee $DIR/err.log
 
           RUN_TEST_EOF
@@ -131,8 +128,6 @@ jobs:
 
       - name: Check result
         id: check
-        env:
-          IS_OPTIONAL: ${{ inputs.IS_OPTIONAL }}
         run: |
           cat $DIR/err.log
 
@@ -142,17 +137,13 @@ jobs:
           potential_infra_failure=$(cat $DIR/err.log | grep -Eqiw "device" && echo true || echo false)
           echo "potential_infra_failure=$potential_infra_failure" >> "$GITHUB_OUTPUT"
 
-          if docker exec nemo_container_${{ github.run_id }} coverage xml; then
-            coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen)
-            echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
+          docker exec nemo_container_${{ github.run_id }} coverage combine
+          docker exec nemo_container_${{ github.run_id }} coverage xml
+          docker cp nemo_container_${{ github.run_id }}:/workspace/.coverage $DIR/.coverage 
+          docker cp nemo_container_${{ github.run_id }}:/workspace/coverage.xml $DIR/coverage.xml
 
-            docker cp nemo_container_${{ github.run_id }}:/workspace/.coverage $DIR/.coverage 
-            docker cp nemo_container_${{ github.run_id }}:/workspace/coverage.xml $DIR/coverage.xml
-          else
-            coverage_report="none"
-            echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
-
-          fi
+          coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen)
+          echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
 
           IS_SUCCESS=$(tail -n 1 $DIR/err.log | grep -q "Finished successfully." && echo "true" || echo "false")
 
@@ -168,6 +159,11 @@ jobs:
 
           exit $EXIT_CODE
 
+      - name: Test coverage
+        shell: bash -x -e -u -o pipefail {0}
+        run: |
+          docker exec -t nemo_container_${{ github.run_id }} coverage report -i
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         if: ${{ steps.check.outputs.coverage_report != 'none' }}