{AKS} Update unit test check-in pipeline

src/aks-preview/azcli_aks_live_test/configs/cli_coverage_config.json

@@ -0,0 +1,16 @@
+{
+    "totals": {
+        "fail_percent": 70,
+        "warning_percent": 80,
+        "fail_missing_lines": null,
+        "warning_missing_lines": null
+    },
+    "files": {
+        "decorator.py": {
+            "fail_percent": 99.5,
+            "warning_percent": 100,
+            "fail_missing_lines": 5,
+            "warning_missing_lines": 0
+        }
+    }
+}

src/aks-preview/azcli_aks_live_test/configs/ext_coverage_config.json

@@ -0,0 +1,16 @@
+{
+    "totals": {
+        "fail_percent": 59,
+        "warning_percent": 80,
+        "fail_missing_lines": null,
+        "warning_missing_lines": null
+    },
+    "files": {
+        "decorator.py": {
+            "fail_percent": 99.5,
+            "warning_percent": 100,
+            "fail_missing_lines": 5,
+            "warning_missing_lines": 0
+        }
+    }
+}

src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh

@@ -48,7 +48,7 @@ setupAZ(){
 # need to be executed in a venv
 installTestPackages(){
     # install pytest plugins
-    pip install pytest-json-report pytest-rerunfailures --upgrade
+    pip install pytest-json-report pytest-rerunfailures pytest-cov --upgrade
 
     # install coverage for measuring code coverage
     pip install coverage

src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh

@@ -44,7 +44,7 @@ az account show
 
 # prepare running options
 # pytest options
-pytest_options="--report-path ./reports"
+pytest_options="-c --report-path ./reports"
 # parallel
 if [[ -n ${PARALLELISM} ]]; then
     pytest_options+=" -j ${PARALLELISM}"
@@ -77,7 +77,7 @@ if [[ -n ${LAST_FAILED} ]]; then
 fi
 
 # filter options
-filter_options=" -c"
+filter_options=""
 # cli matrix
 if [[ -n ${CLI_TEST_MATRIX} ]]; then
     filter_options+=" -cm ./configs/${CLI_TEST_MATRIX}"

src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh

@@ -9,6 +9,8 @@ set -o xtrace
 # check var
 [[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1)
 [[ -z "${ACS_BASE_DIR}" ]] && (echo "ACS_BASE_DIR is empty"; exit 1)
+[[ -z "${IGNORE_EXIT_CODE}" ]] && (echo "IGNORE_EXIT_CODE is empty")
+[[ -z "${CLI_COVERAGE_CONFIG}" ]] && (echo "CLI_COVERAGE_CONFIG is empty")
 
 # activate virtualenv
 source azEnv/bin/activate
@@ -18,25 +20,36 @@ source ./scripts/setup_venv.sh
 removeAKSPreview
 
 # unit test & coverage report
-acs_unit_test_failed=""
 pushd ${ACS_BASE_DIR}
+
 # clean existing coverage report
 (coverage combine || true) && (coverage erase || true)
+
 # perform unit test with module 'unittest'
-# since recording test (performed in test_cli_live.sh) is based on module 'pytest', so skip here
-# coverage run --source=. --omit=*/tests/* -p -m pytest
-if ! coverage run --source=. --omit=*/tests/* -p -m unittest discover; then
-    acs_unit_test_failed="true"
-fi
-# generate & copy coverage report
-coverage combine
+test_result=0
+coverage run --source=. --omit=*/tests/* -m pytest tests/latest/ || test_result=$?
+
+# generate coverage report
+coverage combine || true
 coverage report -m
 coverage json -o coverage_acs.json
 popd
+
+# copy coverage report
 mkdir -p reports/ && cp ${ACS_BASE_DIR}/coverage_acs.json reports/
 
-if [[ ${acs_unit_test_failed} == "true" ]]; then
-    echo "Unit test failed!"
-    echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines."
-    exit 1
+# prepare running options
+# unit test result
+options="--unit-test-result ${test_result} --coverage-report ${ACS_BASE_DIR}/coverage_acs.json"
+# ignore exit code
+if [[ -n ${IGNORE_EXIT_CODE} ]]; then
+    options+=" --ignore-exit-code"
+fi
+if [[ -n ${CLI_COVERAGE_CONFIG} ]]; then
+    options+=" --coverage-config ./configs/${CLI_COVERAGE_CONFIG}"
 fi
+
+combined_result=0
+azaks-cov ${options} || combined_result=$?
+echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines."
+exit ${combined_result}

src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh

@@ -48,7 +48,7 @@ az account show
 if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "only" ]]; then
     # prepare running options
     # pytest options
-    pytest_options="--report-path ./reports"
+    pytest_options="-e --report-path ./reports"
     # parallel
     if [[ -n ${PARALLELISM} ]]; then
         pytest_options+=" -j ${PARALLELISM}"
@@ -81,7 +81,7 @@ if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "on
     fi
 
     # filter options
-    filter_options=" -e"
+    filter_options=""
     # ext matrix
     if [[ -n ${EXT_TEST_MATRIX} ]]; then
         filter_options+=" -em ./configs/${EXT_TEST_MATRIX}"
@@ -179,7 +179,7 @@ fi
 if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then
     # prepare running options
     # pytest options
-    pytest_options="--report-path ./reports"
+    pytest_options="-c --report-path ./reports"
     # parallel
     if [[ -n ${PARALLELISM} ]]; then
         pytest_options+=" -j ${PARALLELISM}"
@@ -212,7 +212,7 @@ if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then
     fi
 
     # filter options
-    filter_options=" -c"
+    filter_options=""
     # cli matrix
     if [[ -n ${CLI_TEST_MATRIX} ]]; then
         filter_options+=" -cm ./configs/${CLI_TEST_MATRIX}"

src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh

@@ -9,6 +9,8 @@ set -o xtrace
 # check var
 [[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1)
 [[ -z "${AKS_PREVIEW_BASE_DIR}" ]] && (echo "AKS_PREVIEW_BASE_DIR is empty"; exit 1)
+[[ -z "${IGNORE_EXIT_CODE}" ]] && (echo "IGNORE_EXIT_CODE is empty")
+[[ -z "${EXT_COVERAGE_CONFIG}" ]] && (echo "CLI_COVERAGE_CONFIG is empty")
 
 # activate virtualenv
 source azEnv/bin/activate
@@ -17,25 +19,36 @@ source azEnv/bin/activate
 ./scripts/setup_venv.sh setup-akspreview
 
 # unit test & coverage report
-azext_aks_preview_unit_test_failed=""
 pushd ${AKS_PREVIEW_BASE_DIR}
+
 # clean existing coverage report
 (coverage combine || true) && (coverage erase || true)
+
 # perform unit test with module 'unittest'
-# since recording test (performed in test_ext_live.sh) is based on module 'pytest', so skip here
-# coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest
-if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest discover; then
-    azext_aks_preview_unit_test_failed="true"
-fi
-# generate & copy coverage report
-coverage combine
+test_result=0
+coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -m pytest tests/latest/ || test_result=$?
+
+# generate coverage report
+coverage combine || true
 coverage report -m
 coverage json -o coverage_azext_aks_preview.json
 popd
+
+# copy coverage report
 mkdir -p reports/ && cp ${AKS_PREVIEW_BASE_DIR}/coverage_azext_aks_preview.json reports/
 
-if [[ ${azext_aks_preview_unit_test_failed} == "true" ]]; then
-    echo "Unit test failed!"
-    echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines."
-    exit 1
+# prepare running options
+# unit test result
+options="--unit-test-result ${test_result} --coverage-report ${AKS_PREVIEW_BASE_DIR}/coverage_azext_aks_preview.json"
+# ignore exit code
+if [[ -n ${IGNORE_EXIT_CODE} ]]; then
+    options+=" --ignore-exit-code"
+fi
+if [[ -n ${EXT_COVERAGE_CONFIG} ]]; then
+    options+=" --coverage-config ./configs/${EXT_COVERAGE_CONFIG}"
 fi
+
+combined_result=0
+azaks-cov ${options} || combined_result=$?
+echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines."
+exit ${combined_result}

src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh

@@ -51,11 +51,13 @@ set -o xtrace
 [[ -z "${SELECTION_MODE}" ]] && (echo "SELECTION_MODE is empty")
 [[ -z "${SELECTION_COUNT}" ]] && (echo "SELECTION_COUNT is empty")
 [[ -z "${UNFIX_RANDOM_SEED}" ]] && (echo "UNFIX_RANDOM_SEED is empty")
-[[ -z "${BACKWARD_COMPATIBILITY_TEST}" ]] && (echo "BACKWARD_COMPATIBILITY_TEST is empty")
+[[ -z "${CLI_COVERAGE_CONFIG}" ]] && (echo "CLI_COVERAGE_CONFIG is empty")
+[[ -z "${EXT_COVERAGE_CONFIG}" ]] && (echo "EXT_COVERAGE_CONFIG is empty")
 [[ -z "${CLI_REPO}" ]] && (echo "CLI_REPO is empty"; exit 1)
 [[ -z "${CLI_BRANCH}" ]] && (echo "CLI_BRANCH is empty"; exit 1)
 [[ -z "${EXT_REPO}" ]] && (echo "EXT_REPO is empty"; exit 1)
 [[ -z "${EXT_BRANCH}" ]] && (echo "EXT_BRANCH is empty"; exit 1)
+[[ -z "${BACKWARD_COMPATIBILITY_TEST}" ]] && (echo "BACKWARD_COMPATIBILITY_TEST is empty")
 
 # base directories for acs, aks-preview and live test
 [[ -z "${ACS_BASE_DIR}" ]] && (echo "ACS_BASE_DIR is empty"; exit 1)
@@ -100,6 +102,7 @@ echo "AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION=${TEST_LOCATION}" >> env.list
 
 # custom - az-aks-tool
 echo "IGNORE_EXIT_CODE=${IGNORE_EXIT_CODE}" >> env.list
+# live test
 echo "COVERAGE=${COVERAGE}" >> env.list
 echo "TEST_MODE=${TEST_MODE}" >> env.list
 echo "CLI_TEST_MATRIX=${CLI_TEST_MATRIX}" >> env.list
@@ -112,6 +115,9 @@ echo "ENABLE_SELECTION=${ENABLE_SELECTION}" >> env.list
 echo "SELECTION_MODE=${SELECTION_MODE}" >> env.list
 echo "SELECTION_COUNT=${SELECTION_COUNT}" >> env.list
 echo "UNFIX_RANDOM_SEED=${UNFIX_RANDOM_SEED}" >> env.list
+# unit test
+echo "CLI_COVERAGE_CONFIG=${CLI_COVERAGE_CONFIG}" >> env.list
+echo "EXT_COVERAGE_CONFIG=${EXT_COVERAGE_CONFIG}" >> env.list
 
 # custom - repo
 echo "CLI_REPO=${CLI_REPO}" >> env.list

src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py

@@ -3195,4 +3195,4 @@ def test_aks_update_label_msi(self, resource_group, resource_group_location):
             self.check('provisioningState', 'Succeeded'),
             self.check('agentPoolProfiles[0].nodeLabels.label1', 'value11'),
             self.check('agentPoolProfiles[0].nodeLabels.label2', None),
-        ])
+        ])