diff --git a/src/aks-preview/azcli_aks_live_test/configs/cli_coverage_config.json b/src/aks-preview/azcli_aks_live_test/configs/cli_coverage_config.json new file mode 100644 index 00000000000..f396afa20ce --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/configs/cli_coverage_config.json @@ -0,0 +1,16 @@ +{ + "totals": { + "fail_percent": 70, + "warning_percent": 80, + "fail_missing_lines": null, + "warning_missing_lines": null + }, + "files": { + "decorator.py": { + "fail_percent": 99.5, + "warning_percent": 100, + "fail_missing_lines": 5, + "warning_missing_lines": 0 + } + } +} \ No newline at end of file diff --git a/src/aks-preview/azcli_aks_live_test/configs/ext_coverage_config.json b/src/aks-preview/azcli_aks_live_test/configs/ext_coverage_config.json new file mode 100644 index 00000000000..da893305895 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/configs/ext_coverage_config.json @@ -0,0 +1,16 @@ +{ + "totals": { + "fail_percent": 59, + "warning_percent": 80, + "fail_missing_lines": null, + "warning_missing_lines": null + }, + "files": { + "decorator.py": { + "fail_percent": 99.5, + "warning_percent": 100, + "fail_missing_lines": 5, + "warning_missing_lines": 0 + } + } +} \ No newline at end of file diff --git a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh index d17324d1775..eb2210a39e1 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh @@ -48,7 +48,7 @@ setupAZ(){ # need to be executed in a venv installTestPackages(){ # install pytest plugins - pip install pytest-json-report pytest-rerunfailures --upgrade + pip install pytest-json-report pytest-rerunfailures pytest-cov --upgrade # install coverage for measuring code coverage pip install coverage diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh index a81efc06a4e..195ed25e960 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh @@ -44,7 +44,7 @@ az account show # prepare running options # pytest options -pytest_options="--report-path ./reports" +pytest_options="-c --report-path ./reports" # parallel if [[ -n ${PARALLELISM} ]]; then pytest_options+=" -j ${PARALLELISM}" @@ -77,7 +77,7 @@ if [[ -n ${LAST_FAILED} ]]; then fi # filter options -filter_options=" -c" +filter_options="" # cli matrix if [[ -n ${CLI_TEST_MATRIX} ]]; then filter_options+=" -cm ./configs/${CLI_TEST_MATRIX}" diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh index 37e799f9f08..a92d5a8b795 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh @@ -9,6 +9,8 @@ set -o xtrace # check var [[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1) [[ -z "${ACS_BASE_DIR}" ]] && (echo "ACS_BASE_DIR is empty"; exit 1) +[[ -z "${IGNORE_EXIT_CODE}" ]] && (echo "IGNORE_EXIT_CODE is empty") +[[ -z "${CLI_COVERAGE_CONFIG}" ]] && (echo "CLI_COVERAGE_CONFIG is empty") # activate virtualenv source azEnv/bin/activate @@ -18,25 +20,36 @@ source ./scripts/setup_venv.sh removeAKSPreview # unit test & coverage report -acs_unit_test_failed="" pushd ${ACS_BASE_DIR} + # clean existing coverage report (coverage combine || true) && (coverage erase || true) + # perform unit test with module 'unittest' -# since recording test (performed in test_cli_live.sh) is based on module 'pytest', so skip here -# coverage run --source=. --omit=*/tests/* -p -m pytest -if ! coverage run --source=. --omit=*/tests/* -p -m unittest discover; then - acs_unit_test_failed="true" -fi -# generate & copy coverage report -coverage combine +test_result=0 +coverage run --source=. --omit=*/tests/* -m pytest tests/latest/ || test_result=$? + +# generate coverage report +coverage combine || true coverage report -m coverage json -o coverage_acs.json popd + +# copy coverage report mkdir -p reports/ && cp ${ACS_BASE_DIR}/coverage_acs.json reports/ -if [[ ${acs_unit_test_failed} == "true" ]]; then - echo "Unit test failed!" - echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." - exit 1 +# prepare running options +# unit test result +options="--unit-test-result ${test_result} --coverage-report ${ACS_BASE_DIR}/coverage_acs.json" +# ignore exit code +if [[ -n ${IGNORE_EXIT_CODE} ]]; then + options+=" --ignore-exit-code" +fi +if [[ -n ${CLI_COVERAGE_CONFIG} ]]; then + options+=" --coverage-config ./configs/${CLI_COVERAGE_CONFIG}" fi + +combined_result=0 +azaks-cov ${options} || combined_result=$? +echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." +exit ${combined_result} diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh index 955168e423e..7ed4c798c8c 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh @@ -48,7 +48,7 @@ az account show if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "only" ]]; then # prepare running options # pytest options - pytest_options="--report-path ./reports" + pytest_options="-e --report-path ./reports" # parallel if [[ -n ${PARALLELISM} ]]; then pytest_options+=" -j ${PARALLELISM}" @@ -81,7 +81,7 @@ if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "on fi # filter options - filter_options=" -e" + filter_options="" # ext matrix if [[ -n ${EXT_TEST_MATRIX} ]]; then filter_options+=" -em ./configs/${EXT_TEST_MATRIX}" @@ -179,7 +179,7 @@ fi if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then # prepare running options # pytest options - pytest_options="--report-path ./reports" + pytest_options="-c --report-path ./reports" # parallel if [[ -n ${PARALLELISM} ]]; then pytest_options+=" -j ${PARALLELISM}" @@ -212,7 +212,7 @@ if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then fi # filter options - filter_options=" -c" + filter_options="" # cli matrix if [[ -n ${CLI_TEST_MATRIX} ]]; then filter_options+=" -cm ./configs/${CLI_TEST_MATRIX}" diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh index 82298d71376..0de60b8018c 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh @@ -9,6 +9,8 @@ set -o xtrace # check var [[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1) [[ -z "${AKS_PREVIEW_BASE_DIR}" ]] && (echo "AKS_PREVIEW_BASE_DIR is empty"; exit 1) +[[ -z "${IGNORE_EXIT_CODE}" ]] && (echo "IGNORE_EXIT_CODE is empty") +[[ -z "${EXT_COVERAGE_CONFIG}" ]] && (echo "CLI_COVERAGE_CONFIG is empty") # activate virtualenv source azEnv/bin/activate @@ -17,25 +19,36 @@ source azEnv/bin/activate ./scripts/setup_venv.sh setup-akspreview # unit test & coverage report -azext_aks_preview_unit_test_failed="" pushd ${AKS_PREVIEW_BASE_DIR} + # clean existing coverage report (coverage combine || true) && (coverage erase || true) + # perform unit test with module 'unittest' -# since recording test (performed in test_ext_live.sh) is based on module 'pytest', so skip here -# coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest -if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest discover; then - azext_aks_preview_unit_test_failed="true" -fi -# generate & copy coverage report -coverage combine +test_result=0 +coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -m pytest tests/latest/ || test_result=$? + +# generate coverage report +coverage combine || true coverage report -m coverage json -o coverage_azext_aks_preview.json popd + +# copy coverage report mkdir -p reports/ && cp ${AKS_PREVIEW_BASE_DIR}/coverage_azext_aks_preview.json reports/ -if [[ ${azext_aks_preview_unit_test_failed} == "true" ]]; then - echo "Unit test failed!" - echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." - exit 1 +# prepare running options +# unit test result +options="--unit-test-result ${test_result} --coverage-report ${AKS_PREVIEW_BASE_DIR}/coverage_azext_aks_preview.json" +# ignore exit code +if [[ -n ${IGNORE_EXIT_CODE} ]]; then + options+=" --ignore-exit-code" +fi +if [[ -n ${EXT_COVERAGE_CONFIG} ]]; then + options+=" --coverage-config ./configs/${EXT_COVERAGE_CONFIG}" fi + +combined_result=0 +azaks-cov ${options} || combined_result=$? +echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." +exit ${combined_result} diff --git a/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh index 62275fcc9a5..b15b99f5d33 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh @@ -51,11 +51,13 @@ set -o xtrace [[ -z "${SELECTION_MODE}" ]] && (echo "SELECTION_MODE is empty") [[ -z "${SELECTION_COUNT}" ]] && (echo "SELECTION_COUNT is empty") [[ -z "${UNFIX_RANDOM_SEED}" ]] && (echo "UNFIX_RANDOM_SEED is empty") -[[ -z "${BACKWARD_COMPATIBILITY_TEST}" ]] && (echo "BACKWARD_COMPATIBILITY_TEST is empty") +[[ -z "${CLI_COVERAGE_CONFIG}" ]] && (echo "CLI_COVERAGE_CONFIG is empty") +[[ -z "${EXT_COVERAGE_CONFIG}" ]] && (echo "EXT_COVERAGE_CONFIG is empty") [[ -z "${CLI_REPO}" ]] && (echo "CLI_REPO is empty"; exit 1) [[ -z "${CLI_BRANCH}" ]] && (echo "CLI_BRANCH is empty"; exit 1) [[ -z "${EXT_REPO}" ]] && (echo "EXT_REPO is empty"; exit 1) [[ -z "${EXT_BRANCH}" ]] && (echo "EXT_BRANCH is empty"; exit 1) +[[ -z "${BACKWARD_COMPATIBILITY_TEST}" ]] && (echo "BACKWARD_COMPATIBILITY_TEST is empty") # base directories for acs, aks-preview and live test [[ -z "${ACS_BASE_DIR}" ]] && (echo "ACS_BASE_DIR is empty"; exit 1) @@ -100,6 +102,7 @@ echo "AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION=${TEST_LOCATION}" >> env.list # custom - az-aks-tool echo "IGNORE_EXIT_CODE=${IGNORE_EXIT_CODE}" >> env.list +# live test echo "COVERAGE=${COVERAGE}" >> env.list echo "TEST_MODE=${TEST_MODE}" >> env.list echo "CLI_TEST_MATRIX=${CLI_TEST_MATRIX}" >> env.list @@ -112,6 +115,9 @@ echo "ENABLE_SELECTION=${ENABLE_SELECTION}" >> env.list echo "SELECTION_MODE=${SELECTION_MODE}" >> env.list echo "SELECTION_COUNT=${SELECTION_COUNT}" >> env.list echo "UNFIX_RANDOM_SEED=${UNFIX_RANDOM_SEED}" >> env.list +# unit test +echo "CLI_COVERAGE_CONFIG=${CLI_COVERAGE_CONFIG}" >> env.list +echo "EXT_COVERAGE_CONFIG=${EXT_COVERAGE_CONFIG}" >> env.list # custom - repo echo "CLI_REPO=${CLI_REPO}" >> env.list diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index eadc80d0aa5..f6933c06efc 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -3195,4 +3195,4 @@ def test_aks_update_label_msi(self, resource_group, resource_group_location): self.check('provisioningState', 'Succeeded'), self.check('agentPoolProfiles[0].nodeLabels.label1', 'value11'), self.check('agentPoolProfiles[0].nodeLabels.label2', None), - ]) \ No newline at end of file + ])