From 0f6b778767839f0930e811a186e0d1662db52df8 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 12 Jul 2023 14:31:44 +0800 Subject: [PATCH 01/33] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7f93385244a..0a413400851 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,7 +76,7 @@ jobs: displayName: 'Use Python 3.6' inputs: versionSpec: 3.6 - - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8==3.5.0 requests + - bash: pip install wheel==0.30.0 pylint==2.17 flake8==6.0 requests displayName: 'Install wheel, pylint, flake8, requests' - bash: python scripts/ci/source_code_static_analysis.py displayName: "Static Analysis" From aa2962e31f91d2bc655eefff228a4349c8aa83d3 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 12 Jul 2023 14:37:07 +0800 Subject: [PATCH 02/33] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0a413400851..ed153fb2e5e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,7 +76,7 @@ jobs: displayName: 'Use Python 3.6' inputs: versionSpec: 3.6 - - bash: pip install wheel==0.30.0 pylint==2.17 flake8==6.0 requests + - bash: pip install wheel==0.30.0 pylint>=2.12,<3 flake8==6.0.0 requests displayName: 'Install wheel, pylint, flake8, requests' - bash: python scripts/ci/source_code_static_analysis.py displayName: "Static Analysis" From 5d49c1968ca1779d5f0cd4afe282ab21ffdce3f6 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 12 Jul 2023 15:09:35 +0800 Subject: [PATCH 03/33] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ed153fb2e5e..b5da138503c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,7 +76,7 @@ jobs: displayName: 'Use Python 3.6' inputs: versionSpec: 3.6 - - bash: pip install wheel==0.30.0 pylint>=2.12,<3 flake8==6.0.0 requests + - bash: pip install wheel==0.30.0 "pylint>=2.12,<3" flake8==6.0.0 requests displayName: 'Install wheel, pylint, flake8, requests' - bash: python scripts/ci/source_code_static_analysis.py displayName: "Static Analysis" From 03b4199b5f52601e17b9618a5680dd694b71b3b1 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 12 Jul 2023 15:12:27 +0800 Subject: [PATCH 04/33] Update azure-pipelines.yml --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b5da138503c..bdec3e8beb2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -73,9 +73,9 @@ jobs: name: 'pool-ubuntu-2004' steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' + displayName: 'Use Python 3.10' inputs: - versionSpec: 3.6 + versionSpec: 3.10 - bash: pip install wheel==0.30.0 "pylint>=2.12,<3" flake8==6.0.0 requests displayName: 'Install wheel, pylint, flake8, requests' - bash: python scripts/ci/source_code_static_analysis.py From cbf7b3df958110528514042bbaee8f7e04b32c30 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 3 Aug 2023 15:03:21 +0800 Subject: [PATCH 05/33] Update azure-pipelines.yml --- azure-pipelines.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bdec3e8beb2..c0f9fe7ca14 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,6 +81,22 @@ jobs: - bash: python scripts/ci/source_code_static_analysis.py displayName: "Static Analysis" +- job: StaticAnalysis v2 + displayName: "Static Analysis v2" + pool: + name: 'pool-ubuntu-2004' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev style + displayName: "Static Analysis" + - job: IndexVerify displayName: "Verify Extensions Index" pool: From 8a7f42b78b3feb0ef30d44ba4b7be933e4c7dffb Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 3 Aug 2023 15:06:50 +0800 Subject: [PATCH 06/33] update --- .azure-pipelines/templates/azdev_setup.yml | 3 ++- azure-pipelines.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index 71084999a01..ae5a8145e76 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -14,7 +14,8 @@ steps: # clone azure-cli git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli python -m pip install -U pip - pip install azdev + pip install git+https://github.com/dciborow/azure-cli-dev-tools.git@dciborow/pylint + # pip install azdev azdev --version azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug pip list -v diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c0f9fe7ca14..43165bd8c55 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -92,8 +92,9 @@ jobs: versionSpec: 3.10 - template: .azure-pipelines/templates/azdev_setup.yml - bash: | + #!/usr/bin/env bash set -ev - . env/bin/activate + source ./env/bin/activate azdev style displayName: "Static Analysis" From 46ae8fbb5f56aeb4d5367f3a0dfc32f05822cf94 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Aug 2023 09:10:20 +0800 Subject: [PATCH 07/33] Empty From 158dec144edd68bd8cf76cabb85a6de71423f907 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 11 Aug 2023 09:42:41 +0800 Subject: [PATCH 08/33] Update azure-pipelines.yml --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f2b7a3f880e..71933f265f9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,7 +81,7 @@ jobs: - bash: python scripts/ci/source_code_static_analysis.py displayName: "Static Analysis" -- job: StaticAnalysis v2 +- job: StaticAnalysisv2 displayName: "Static Analysis v2" pool: name: 'pool-ubuntu-2004' @@ -96,7 +96,7 @@ jobs: set -ev source ./env/bin/activate azdev style - displayName: "Static Analysis" + displayName: "Static Analysis v2" - job: IndexVerify displayName: "Verify Extensions Index" From 26131755023e1d9868298840718935829bc00180 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 11 Aug 2023 10:22:05 +0800 Subject: [PATCH 09/33] Update azdev_setup.yml --- .azure-pipelines/templates/azdev_setup.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index ae5a8145e76..71084999a01 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -14,8 +14,7 @@ steps: # clone azure-cli git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli python -m pip install -U pip - pip install git+https://github.com/dciborow/azure-cli-dev-tools.git@dciborow/pylint - # pip install azdev + pip install azdev azdev --version azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug pip list -v From 3568713e1a3f1d1c97a232c882614700641ef921 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 11 Aug 2023 11:18:54 +0800 Subject: [PATCH 10/33] Update azure-pipelines.yml --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 71933f265f9..818f47f027f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -95,7 +95,8 @@ jobs: #!/usr/bin/env bash set -ev source ./env/bin/activate - azdev style + azdev style --debug --verbose + # azdev style --repo azure-cli --tgt upstream/master --src upstream/dev displayName: "Static Analysis v2" - job: IndexVerify From 377f4f071dae95deee649769f85b1da7174329ea Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 11 Aug 2023 14:24:45 +0800 Subject: [PATCH 11/33] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 818f47f027f..70cf76805c5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -95,7 +95,7 @@ jobs: #!/usr/bin/env bash set -ev source ./env/bin/activate - azdev style --debug --verbose + azdev style EXT --debug --verbose # azdev style --repo azure-cli --tgt upstream/master --src upstream/dev displayName: "Static Analysis v2" From bf31b04ef4078f7fa6aa8e0e41a4286001a6fcdd Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 23 Nov 2023 14:46:46 +0800 Subject: [PATCH 12/33] Empty From d0055e447f3759879cd63309eef3e28049da6e36 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 23 Nov 2023 15:54:15 +0800 Subject: [PATCH 13/33] update --- .flake8 | 22 ++++-- pylintrc | 75 +++++++++++------- scripts/ci/azdev_style_check.py | 134 ++++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 38 deletions(-) create mode 100644 scripts/ci/azdev_style_check.py diff --git a/.flake8 b/.flake8 index be946cee0b0..f62e177f7ac 100644 --- a/.flake8 +++ b/.flake8 @@ -2,14 +2,20 @@ max-line-length = 120 max-complexity = 10 ignore = - E501, # line too long, it is covered by pylint - E722, # bare except, bad practice, to be removed in the future - F401, # imported but unused, too many violations, to be removed in the future - F811, # redefinition of unused, to be removed in the future - C901 # code flow is too complex, too many violations, to be removed in the future - W503 # line break before binary operator effect on readability is subjective - W504 # line break after binary operator effect on readability is subjective - + # line too long, it is covered by pylint + E501 + # bare except, bad practice, to be removed in the future + E722 + # imported but unused, too many violations, to be removed in the future + F401 + # redefinition of unused, to be removed in the future + F811 + # code flow is too complex, too many violations, to be removed in the future + C901 + # line break before binary operator effect on readability is subjective + W503 + # line break after binary operator effect on readability is subjective + W504 exclude = */vendored_sdks docs diff --git a/pylintrc b/pylintrc index dbe731101ce..6c4a8f4c7ec 100644 --- a/pylintrc +++ b/pylintrc @@ -1,27 +1,58 @@ [MASTER] ignore=tests,generated,vendored_sdks,privates +ignore-patterns=test.*,azure_devops_build.* +ignore-paths=.*/aaz reports=no [MESSAGES CONTROL] # For all codes, run 'pylint --list-msgs' or go to 'https://pylint.readthedocs.io/en/latest/reference_guide/features.html' +# cyclic-import: because of https://github.com/PyCQA/pylint/issues/850 +# import-outside-toplevel: Lazy import to improve performance # locally-disabled: Warning locally suppressed using disable-msg -# cyclic-import: Because of https://github.com/PyCQA/pylint/issues/850 # too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods. -# import-outside-toplevel: Lazy import to improve performance +# consider-using-f-string: str.format is still a Python standard +# unspecified-encoding: Allow using the system encoding, instead of forcing utf-8 to avoid opening errors disable= - missing-docstring, - locally-disabled, - fixme, + arguments-out-of-order, + bad-option-value, + consider-using-with, cyclic-import, - too-many-arguments, - invalid-name, duplicate-code, + fixme, import-outside-toplevel, - too-many-lines - -[TYPECHECK] -# For Azure CLI extensions, we ignore some import errors as they'll be available in the environment of the CLI -ignored-modules=azure,azure.cli,azure.cli.core,azure.cli.core.commands,knack,msrestazure,argcomplete,azure.cli.testsdk,isodate,OpenSSL + inconsistent-return-statements, + invalid-name, + invalid-str-returned, + locally-disabled, + missing-docstring, + missing-kwoa, + no-member, + no-value-for-parameter, + raise-missing-from, + subprocess-run-check, + super-with-arguments, + too-many-arguments, + too-many-function-args, + too-many-lines, + using-constant-test, + wrong-import-order, + consider-using-f-string, + unspecified-encoding, + use-maxsplit-arg, + arguments-renamed, + consider-using-in, + use-dict-literal, + consider-using-dict-items, + consider-using-enumerate, + redundant-u-string-prefix, + raising-bad-type, + unused-private-member, + used-before-assignment, + # These rules were added in Pylint >= 2.12, disable them to avoid making retroactive change + missing-timeout, + superfluous-parens, + implicit-str-concat, + unnecessary-dunder-call, [FORMAT] max-line-length=120 @@ -35,24 +66,8 @@ init-import=yes max-locals=25 # Maximum number of branch for function / method body max-branches=20 +# Temporally disable this check as inline disable is not working, see https://github.com/pylint-dev/pylint/issues/8806 +max-statements=500 [SIMILARITIES] min-similarity-lines=10 - -[BASIC] -# Naming hints based on PEP 8 (https://www.python.org/dev/peps/pep-0008/#naming-conventions). -# Consider these guidelines and not hard rules. Read PEP 8 for more details. - -# The invalid-name checker must be **enabled** for these hints to be used. -include-naming-hint=yes - -module-name-hint=lowercase (keep short; underscores are discouraged) -const-name-hint=UPPER_CASE_WITH_UNDERSCORES -class-name-hint=CapitalizedWords -class-attribute-name-hint=lower_case_with_underscores -attr-name-hint=lower_case_with_underscores -method-name-hint=lower_case_with_underscores -function-name-hint=lower_case_with_underscores -argument-name-hint=lower_case_with_underscores -variable-name-hint=lower_case_with_underscores -inlinevar-name-hint=lower_case_with_underscores (short is OK) diff --git a/scripts/ci/azdev_style_check.py b/scripts/ci/azdev_style_check.py new file mode 100644 index 00000000000..d3983606fb1 --- /dev/null +++ b/scripts/ci/azdev_style_check.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import json +import logging +import os +import subprocess +import sys + +from util import diff_code + +from azdev.utilities.path import get_cli_repo_path, get_ext_repo_paths +from subprocess import run + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +logger.addHandler(ch) + +pull_request_number = os.environ.get('PULL_REQUEST_NUMBER', None) +job_name = os.environ.get('JOB_NAME', None) +azdev_test_result_dir = os.path.expanduser("~/.azdev/env_config/mnt/vss/_work/1/s/env") +src_branch = os.environ.get('PR_TARGET_BRANCH', None) +target_branch = 'merged_pr' +base_meta_path = '~/_work/1/base_meta' +diff_meta_path = '~/_work/1/diff_meta' +output_path = '~/_work/1/output_meta' + + +def install_extensions(diif_ref, branch): + for tname, ext_path in diif_ref: + ext_name = ext_path.split('/')[-1] + logger.info(f'installing extension: {ext_name}') + cmd = ['azdev', 'extension', 'add', ext_name] + logger.info(f'cmd: {cmd}') + out = run(cmd, capture_output=True, text=True) + if out.returncode and branch == 'base' and 'ERROR: extension(s) not found' in out.stderr: + print(f"{cmd} failed, extesion {ext_name} is not exist on base branch, skip it.") + sys.exit(0) + elif out.returncode: + raise RuntimeError(f"{cmd} failed") + + +def uninstall_extensions(diif_ref): + for tname, ext_path in diif_ref: + ext_name = ext_path.split('/')[-1] + logger.info(f'uninstalling extension: {ext_name}') + cmd = ['azdev', 'extension', 'remove', ext_name] + logger.info(f'cmd: {cmd}') + out = run(cmd) + if out.returncode: + raise RuntimeError(f"{cmd} failed") + + +# def get_diff_meta_files(diff_ref): +# cmd = ['git', 'checkout', '-b', target_branch] +# print(cmd) +# subprocess.run(cmd) +# cmd = ['git', 'checkout', src_branch] +# print(cmd) +# subprocess.run(cmd) +# cmd = ['git', 'checkout', target_branch] +# print(cmd) +# subprocess.run(cmd) +# cmd = ['git', 'rev-parse', 'HEAD'] +# print(cmd) +# subprocess.run(cmd) +# install_extensions(diff_ref, branch='target') +# cmd = ['azdev', 'command-change', 'meta-export', '--src', src_branch, '--tgt', target_branch, '--repo', get_ext_repo_paths()[0], '--meta-output-path', diff_meta_path] +# print(cmd) +# subprocess.run(cmd) +# cmd = ['ls', '-al', diff_meta_path] +# print(cmd) +# subprocess.run(cmd) +# uninstall_extensions(diff_ref) +# +# +# def get_base_meta_files(diff_ref): +# cmd = ['git', 'checkout', src_branch] +# print(cmd) +# subprocess.run(cmd) +# cmd = ['git', 'rev-parse', 'HEAD'] +# print(cmd) +# subprocess.run(cmd) +# install_extensions(diff_ref, branch='base') +# cmd = ['azdev', 'command-change', 'meta-export', 'EXT', '--meta-output-path', base_meta_path] +# print(cmd) +# subprocess.run(cmd) +# cmd = ['ls', '-al', base_meta_path] +# print(cmd) +# subprocess.run(cmd) + + +def meta_diff(): + if os.path.exists(diff_meta_path): + for file in os.listdir(diff_meta_path): + if file.endswith('.json'): + cmd = ['azdev', 'command-change', 'meta-diff', '--base-meta-file', os.path.join(base_meta_path, file), '--diff-meta-file', os.path.join(diff_meta_path, file), '--output-file', os.path.join(output_path, file)] + print(cmd) + subprocess.run(cmd) + cmd = ['ls', '-al', output_path] + print(cmd) + subprocess.run(cmd) + + +def azdev_style_check(repo_path, src, tgt): + for tname, ext_path in diif_ref: + ext_name = ext_path.split('/')[-1] + logger.info(f'installing extension: {ext_name}') + cmd = ['azdev', 'extension', 'add', ext_name] + logger.info(f'cmd: {cmd}') + out = run(cmd, capture_output=True, text=True) + if out.returncode and branch == 'base' and 'ERROR: extension(s) not found' in out.stderr: + print(f"{cmd} failed, extesion {ext_name} is not exist on base branch, skip it.") + sys.exit(0) + elif out.returncode: + raise RuntimeError(f"{cmd} failed") + + +def main(): + if pull_request_number != '$(System.PullRequest.PullRequestNumber)': + logger.info("Start azdev style test ...\n") + diff_ref = diff_code(src_branch, 'HEAD') + install_extensions(diff_ref) + azdev_style_check() + + +if __name__ == '__main__': + main() From 6a2cec868b8960144ce60001ddfbea2fe4044134 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 23 Nov 2023 15:57:23 +0800 Subject: [PATCH 14/33] modify containerapp --- src/containerapp/azext_containerapp/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/containerapp/azext_containerapp/__init__.py b/src/containerapp/azext_containerapp/__init__.py index e0b1eccac7a..45b0764676e 100644 --- a/src/containerapp/azext_containerapp/__init__.py +++ b/src/containerapp/azext_containerapp/__init__.py @@ -1,3 +1,4 @@ +#test # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. From d4f8312093cc96853a2c383cec368a1be17e85db Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 23 Nov 2023 16:19:45 +0800 Subject: [PATCH 15/33] update --- azure-pipelines.yml | 15 ++--- scripts/ci/azdev_style_check.py | 110 ++------------------------------ 2 files changed, 14 insertions(+), 111 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 53e76d15269..c2e5fdaa02d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -73,31 +73,30 @@ jobs: name: 'pool-ubuntu-2004' steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' + displayName: 'Use Python 3.11' inputs: - versionSpec: 3.10 + versionSpec: 3.11 - bash: pip install wheel==0.30.0 "pylint>=2.12,<3" flake8==6.0.0 requests displayName: 'Install wheel, pylint, flake8, requests' - bash: python scripts/ci/source_code_static_analysis.py displayName: "Static Analysis" - job: StaticAnalysisv2 - displayName: "Static Analysis v2" + displayName: "Azdev Static Analysis" pool: name: 'pool-ubuntu-2004' steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' + displayName: 'Use Python 3.11' inputs: - versionSpec: 3.10 + versionSpec: 3.11 - template: .azure-pipelines/templates/azdev_setup.yml - bash: | #!/usr/bin/env bash set -ev source ./env/bin/activate - azdev style EXT --debug --verbose - # azdev style --repo azure-cli --tgt upstream/master --src upstream/dev - displayName: "Static Analysis v2" + python scripts/ci/azdev_style_check.py + displayName: "Azdev Style" - job: IndexVerify displayName: "Verify Extensions Index" diff --git a/scripts/ci/azdev_style_check.py b/scripts/ci/azdev_style_check.py index d3983606fb1..d2faa9c6a18 100644 --- a/scripts/ci/azdev_style_check.py +++ b/scripts/ci/azdev_style_check.py @@ -5,15 +5,9 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import json import logging -import os -import subprocess -import sys - from util import diff_code -from azdev.utilities.path import get_cli_repo_path, get_ext_repo_paths from subprocess import run logger = logging.getLogger(__name__) @@ -22,112 +16,22 @@ ch.setLevel(logging.DEBUG) logger.addHandler(ch) -pull_request_number = os.environ.get('PULL_REQUEST_NUMBER', None) -job_name = os.environ.get('JOB_NAME', None) -azdev_test_result_dir = os.path.expanduser("~/.azdev/env_config/mnt/vss/_work/1/s/env") -src_branch = os.environ.get('PR_TARGET_BRANCH', None) -target_branch = 'merged_pr' -base_meta_path = '~/_work/1/base_meta' -diff_meta_path = '~/_work/1/diff_meta' -output_path = '~/_work/1/output_meta' - -def install_extensions(diif_ref, branch): - for tname, ext_path in diif_ref: +def azdev_style_check(diff_ref): + cmd = ['azdev', 'style'] + for tname, ext_path in diff_ref: ext_name = ext_path.split('/')[-1] - logger.info(f'installing extension: {ext_name}') - cmd = ['azdev', 'extension', 'add', ext_name] + cmd += [ext_name] logger.info(f'cmd: {cmd}') out = run(cmd, capture_output=True, text=True) - if out.returncode and branch == 'base' and 'ERROR: extension(s) not found' in out.stderr: - print(f"{cmd} failed, extesion {ext_name} is not exist on base branch, skip it.") - sys.exit(0) - elif out.returncode: - raise RuntimeError(f"{cmd} failed") - - -def uninstall_extensions(diif_ref): - for tname, ext_path in diif_ref: - ext_name = ext_path.split('/')[-1] - logger.info(f'uninstalling extension: {ext_name}') - cmd = ['azdev', 'extension', 'remove', ext_name] - logger.info(f'cmd: {cmd}') - out = run(cmd) if out.returncode: raise RuntimeError(f"{cmd} failed") -# def get_diff_meta_files(diff_ref): -# cmd = ['git', 'checkout', '-b', target_branch] -# print(cmd) -# subprocess.run(cmd) -# cmd = ['git', 'checkout', src_branch] -# print(cmd) -# subprocess.run(cmd) -# cmd = ['git', 'checkout', target_branch] -# print(cmd) -# subprocess.run(cmd) -# cmd = ['git', 'rev-parse', 'HEAD'] -# print(cmd) -# subprocess.run(cmd) -# install_extensions(diff_ref, branch='target') -# cmd = ['azdev', 'command-change', 'meta-export', '--src', src_branch, '--tgt', target_branch, '--repo', get_ext_repo_paths()[0], '--meta-output-path', diff_meta_path] -# print(cmd) -# subprocess.run(cmd) -# cmd = ['ls', '-al', diff_meta_path] -# print(cmd) -# subprocess.run(cmd) -# uninstall_extensions(diff_ref) -# -# -# def get_base_meta_files(diff_ref): -# cmd = ['git', 'checkout', src_branch] -# print(cmd) -# subprocess.run(cmd) -# cmd = ['git', 'rev-parse', 'HEAD'] -# print(cmd) -# subprocess.run(cmd) -# install_extensions(diff_ref, branch='base') -# cmd = ['azdev', 'command-change', 'meta-export', 'EXT', '--meta-output-path', base_meta_path] -# print(cmd) -# subprocess.run(cmd) -# cmd = ['ls', '-al', base_meta_path] -# print(cmd) -# subprocess.run(cmd) - - -def meta_diff(): - if os.path.exists(diff_meta_path): - for file in os.listdir(diff_meta_path): - if file.endswith('.json'): - cmd = ['azdev', 'command-change', 'meta-diff', '--base-meta-file', os.path.join(base_meta_path, file), '--diff-meta-file', os.path.join(diff_meta_path, file), '--output-file', os.path.join(output_path, file)] - print(cmd) - subprocess.run(cmd) - cmd = ['ls', '-al', output_path] - print(cmd) - subprocess.run(cmd) - - -def azdev_style_check(repo_path, src, tgt): - for tname, ext_path in diif_ref: - ext_name = ext_path.split('/')[-1] - logger.info(f'installing extension: {ext_name}') - cmd = ['azdev', 'extension', 'add', ext_name] - logger.info(f'cmd: {cmd}') - out = run(cmd, capture_output=True, text=True) - if out.returncode and branch == 'base' and 'ERROR: extension(s) not found' in out.stderr: - print(f"{cmd} failed, extesion {ext_name} is not exist on base branch, skip it.") - sys.exit(0) - elif out.returncode: - raise RuntimeError(f"{cmd} failed") - - def main(): - if pull_request_number != '$(System.PullRequest.PullRequestNumber)': - logger.info("Start azdev style test ...\n") - diff_ref = diff_code(src_branch, 'HEAD') - install_extensions(diff_ref) - azdev_style_check() + logger.info("Start azdev style test ...\n") + diff_ref = diff_code('main', 'HEAD') + azdev_style_check(diff_ref) if __name__ == '__main__': From f095fc870c8530d363a1b98847f19b672f31644b Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 23 Nov 2023 16:32:42 +0800 Subject: [PATCH 16/33] Update azdev_style_check.py --- scripts/ci/azdev_style_check.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/ci/azdev_style_check.py b/scripts/ci/azdev_style_check.py index d2faa9c6a18..fe0a2873202 100644 --- a/scripts/ci/azdev_style_check.py +++ b/scripts/ci/azdev_style_check.py @@ -20,8 +20,7 @@ def azdev_style_check(diff_ref): cmd = ['azdev', 'style'] for tname, ext_path in diff_ref: - ext_name = ext_path.split('/')[-1] - cmd += [ext_name] + cmd += [tname] logger.info(f'cmd: {cmd}') out = run(cmd, capture_output=True, text=True) if out.returncode: From c69cc0e2ffd744e21cd69324fdfb64c7d17f5388 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 23 Nov 2023 16:58:41 +0800 Subject: [PATCH 17/33] Update azdev_style_check.py --- scripts/ci/azdev_style_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/azdev_style_check.py b/scripts/ci/azdev_style_check.py index fe0a2873202..e1e2cb528cd 100644 --- a/scripts/ci/azdev_style_check.py +++ b/scripts/ci/azdev_style_check.py @@ -22,7 +22,7 @@ def azdev_style_check(diff_ref): for tname, ext_path in diff_ref: cmd += [tname] logger.info(f'cmd: {cmd}') - out = run(cmd, capture_output=True, text=True) + out = run(cmd, shell=True) if out.returncode: raise RuntimeError(f"{cmd} failed") From b15bde62b5232b7b9649d285b9dbc9e408521e6e Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 23 Nov 2023 17:31:29 +0800 Subject: [PATCH 18/33] Update azdev_style_check.py --- scripts/ci/azdev_style_check.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/ci/azdev_style_check.py b/scripts/ci/azdev_style_check.py index e1e2cb528cd..92bf59ffcd3 100644 --- a/scripts/ci/azdev_style_check.py +++ b/scripts/ci/azdev_style_check.py @@ -18,13 +18,14 @@ def azdev_style_check(diff_ref): - cmd = ['azdev', 'style'] + extensions = '' for tname, ext_path in diff_ref: - cmd += [tname] - logger.info(f'cmd: {cmd}') - out = run(cmd, shell=True) - if out.returncode: - raise RuntimeError(f"{cmd} failed") + extensions += f'{tname} ' + cmd = f'azdev style {extensions} --verbose' + logger.info(f'cmd: {cmd}') + out = run(cmd, shell=True) + if out.returncode: + raise RuntimeError(f"{cmd} failed") def main(): From 38a0f8a2fbfb8a9a05286ef5c8dc1669db66e2ce Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 23 Nov 2023 17:56:41 +0800 Subject: [PATCH 19/33] update --- scripts/ci/azdev_style_check.py | 8 +++----- scripts/ci/verify_linter.py | 5 +++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/ci/azdev_style_check.py b/scripts/ci/azdev_style_check.py index 92bf59ffcd3..ac4017e853e 100644 --- a/scripts/ci/azdev_style_check.py +++ b/scripts/ci/azdev_style_check.py @@ -18,13 +18,11 @@ def azdev_style_check(diff_ref): - extensions = '' - for tname, ext_path in diff_ref: - extensions += f'{tname} ' + extensions = ' '.join(f'{tname}' for tname, _ in diff_ref) cmd = f'azdev style {extensions} --verbose' logger.info(f'cmd: {cmd}') - out = run(cmd, shell=True) - if out.returncode: + process = run(cmd, shell=True, capture_output=True, text=True) + if process.returncode != 0: raise RuntimeError(f"{cmd} failed") diff --git a/scripts/ci/verify_linter.py b/scripts/ci/verify_linter.py index 7fee6294307..bf4f2ac00a3 100644 --- a/scripts/ci/verify_linter.py +++ b/scripts/ci/verify_linter.py @@ -78,6 +78,9 @@ def remove(self): def linter(self): self._cmd('azdev linter --include-whl-extensions {}'.format(self.extension_name)) + def style(self): + self._cmd('azdev style {}'.format(self.extension_name)) + def build(self): pass @@ -161,6 +164,7 @@ def linter_on_external_extension(index_json): azdev_extension = AzdevExtensionHelper(name) azdev_extension.linter() + azdev_extension.style() print('Checking service name for external extensions') service_name.check() @@ -186,6 +190,7 @@ def linter_on_internal_extension(modified_files): azdev_extension = AzdevExtensionHelper(name) azdev_extension.add_from_code() azdev_extension.linter() + azdev_extension.style() print('Checking service name for internal extensions') service_name.check() From 06f48fc6daec88cb4b2add95ef287328431c1aef Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 1 Dec 2023 17:45:50 +0800 Subject: [PATCH 20/33] update --- azure-pipelines.yml | 39 ++----------------- ...verify_linter.py => azdev_linter_style.py} | 18 ++++----- scripts/ci/test_source.py | 9 +++-- 3 files changed, 19 insertions(+), 47 deletions(-) rename scripts/ci/{verify_linter.py => azdev_linter_style.py} (94%) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c2e5fdaa02d..5911c056653 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -67,37 +67,6 @@ jobs: source ./env/bin/activate azdev verify license -- job: StaticAnalysis - displayName: "Static Analysis" - pool: - name: 'pool-ubuntu-2004' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' - inputs: - versionSpec: 3.11 - - bash: pip install wheel==0.30.0 "pylint>=2.12,<3" flake8==6.0.0 requests - displayName: 'Install wheel, pylint, flake8, requests' - - bash: python scripts/ci/source_code_static_analysis.py - displayName: "Static Analysis" - -- job: StaticAnalysisv2 - displayName: "Azdev Static Analysis" - pool: - name: 'pool-ubuntu-2004' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' - inputs: - versionSpec: 3.11 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - #!/usr/bin/env bash - set -ev - source ./env/bin/activate - python scripts/ci/azdev_style_check.py - displayName: "Azdev Style" - - job: IndexVerify displayName: "Verify Extensions Index" pool: @@ -148,8 +117,8 @@ jobs: ADO_PULL_REQUEST_LATEST_COMMIT: HEAD ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) -- job: LintModifiedExtensions - displayName: "CLI Linter on Modified Extensions" +- job: AzdevModifiedExtensions + displayName: "azdev linter && style on Modified Extensions" condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) pool: name: 'pool-ubuntu-2004' @@ -166,8 +135,8 @@ jobs: # overwrite the default AZURE_EXTENSION_DIR set by ADO AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version - AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/verify_linter.py - displayName: "CLI Linter on Modified Extension" + AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/azdev_linter_style.py + displayName: "azdev linter && style on Modified Extensions" env: ADO_PULL_REQUEST_LATEST_COMMIT: HEAD ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) diff --git a/scripts/ci/verify_linter.py b/scripts/ci/azdev_linter_style.py similarity index 94% rename from scripts/ci/verify_linter.py rename to scripts/ci/azdev_linter_style.py index bf4f2ac00a3..5d0b446fecd 100644 --- a/scripts/ci/verify_linter.py +++ b/scripts/ci/azdev_linter_style.py @@ -4,7 +4,7 @@ # -------------------------------------------------------------------------------------------- """ -This script is used to verify linter on extensions. +This script is used to run azdev linter and azdev style on extensions. It's only working on ADO by default. If want to run locally, please update the target branch/commit to find diff in function find_modified_files_against_master_branch() @@ -134,10 +134,10 @@ def contain_extension_code(files): return False -def linter_on_external_extension(index_json): +def azdev_on_external_extension(index_json): """ Check if the modified metadata items in index.json refer to the extension in repo. - If not, az extension linter on wheel. Otherwise skip it. + If not, az extension check on wheel. Otherwise skip it. """ public_extensions = json.loads(check_output('az extension list-available -d', shell=True)) @@ -172,7 +172,7 @@ def linter_on_external_extension(index_json): az_extension.remove() -def linter_on_internal_extension(modified_files): +def azdev_on_internal_extension(modified_files): extension_names = set() for f in modified_files: @@ -182,7 +182,7 @@ def linter_on_internal_extension(modified_files): if not extension_names: separator_line() - print('no extension source code modified, no extension needs to be linter') + print('no extension source code modified, no extension needs to be checked') for name in extension_names: separator_line() @@ -206,8 +206,8 @@ def main(): # This scenarios is for modify index.json only. # If the modified metadata items refer to the extension code exits in this repo, PR is be created via Pipeline. # If the modified metadata items refer to the extension code doesn't exist, PR is created from Service Team. - # We try to verify linter on it. - linter_on_external_extension(modified_files[0]) + # We try to run azdev linter and azdev style on it. + azdev_on_external_extension(modified_files[0]) else: # modified files contain more than one file @@ -216,10 +216,10 @@ def main(): if contain_index_json(modified_files): raise ModifiedFilesNotAllowedError() - linter_on_internal_extension(modified_files) + azdev_on_internal_extension(modified_files) else: separator_line() - print('no extension source code modified, no extension needs to be linter') + print('no extension source code modified, no extension needs to be checked') separator_line() diff --git a/scripts/ci/test_source.py b/scripts/ci/test_source.py index 53aca2ae0a3..3fbb550a6c0 100755 --- a/scripts/ci/test_source.py +++ b/scripts/ci/test_source.py @@ -65,13 +65,16 @@ def run_command(cmd, check_return_code=False, cwd=None): def test_extension(): - for tname, ext_path in ALL_TESTS: + for pkg_name, ext_path in ALL_TESTS: ext_name = ext_path.split('/')[-1] logger.info(f'installing extension: {ext_name}') cmd = ['azdev', 'extension', 'add', ext_name] run_command(cmd, check_return_code=True) - # python -m azdev test --no-exitfirst --discover --verbose azext_containerapp - test_args = [sys.executable, '-m', 'azdev', 'test', '--no-exitfirst', '--discover', '--verbose', tname] + # Use azext_$ext_name, a unique long name for testing, to avoid the following error when the main module and extension name have the same name: + # TODO + # For the recommended azdev test example, please refer to: `azdev test --help` + # `python -m azdev test --no-exitfirst --discover --verbose azext_containerapp` + test_args = [sys.executable, '-m', 'azdev', 'test', '--no-exitfirst', '--discover', '--verbose', pkg_name] logger.warning(f'test_args: {test_args}') run_command(test_args, check_return_code=True) logger.info(f'uninstalling extension: {ext_name}') From d5da72a074fcfcc571dddbe63f1bbebe99e661b5 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 1 Dec 2023 17:46:21 +0800 Subject: [PATCH 21/33] Delete azdev_style_check.py --- scripts/ci/azdev_style_check.py | 36 --------------------------------- 1 file changed, 36 deletions(-) delete mode 100644 scripts/ci/azdev_style_check.py diff --git a/scripts/ci/azdev_style_check.py b/scripts/ci/azdev_style_check.py deleted file mode 100644 index ac4017e853e..00000000000 --- a/scripts/ci/azdev_style_check.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -import logging -from util import diff_code - -from subprocess import run - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -logger.addHandler(ch) - - -def azdev_style_check(diff_ref): - extensions = ' '.join(f'{tname}' for tname, _ in diff_ref) - cmd = f'azdev style {extensions} --verbose' - logger.info(f'cmd: {cmd}') - process = run(cmd, shell=True, capture_output=True, text=True) - if process.returncode != 0: - raise RuntimeError(f"{cmd} failed") - - -def main(): - logger.info("Start azdev style test ...\n") - diff_ref = diff_code('main', 'HEAD') - azdev_style_check(diff_ref) - - -if __name__ == '__main__': - main() From 65c43a0c5487f9217388a842aa2faf1a3d1c6258 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 1 Dec 2023 17:50:15 +0800 Subject: [PATCH 22/33] Update test_source.py --- scripts/ci/test_source.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/ci/test_source.py b/scripts/ci/test_source.py index 3fbb550a6c0..94ddf5bafd7 100755 --- a/scripts/ci/test_source.py +++ b/scripts/ci/test_source.py @@ -70,12 +70,17 @@ def test_extension(): logger.info(f'installing extension: {ext_name}') cmd = ['azdev', 'extension', 'add', ext_name] run_command(cmd, check_return_code=True) + # Use azext_$ext_name, a unique long name for testing, to avoid the following error when the main module and extension name have the same name: - # TODO + # 'containerapp' exists in both 'azext_containerapp' and 'containerapp'. Resolve using `azext_containerapp.containerapp` or `containerapp.containerapp` + # 'containerapp' not found. If newly added, re-run with --discover + # No tests selected to run. + # ---------------------------------------------------------------------- # For the recommended azdev test example, please refer to: `azdev test --help` # `python -m azdev test --no-exitfirst --discover --verbose azext_containerapp` test_args = [sys.executable, '-m', 'azdev', 'test', '--no-exitfirst', '--discover', '--verbose', pkg_name] logger.warning(f'test_args: {test_args}') + run_command(test_args, check_return_code=True) logger.info(f'uninstalling extension: {ext_name}') cmd = ['azdev', 'extension', 'remove', ext_name] From 112e68508847310f4039a32362b70af32acf4190 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 7 Dec 2023 11:46:56 +0800 Subject: [PATCH 23/33] Update azdev_linter_style.py --- scripts/ci/azdev_linter_style.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ci/azdev_linter_style.py b/scripts/ci/azdev_linter_style.py index 5d0b446fecd..21735461d16 100644 --- a/scripts/ci/azdev_linter_style.py +++ b/scripts/ci/azdev_linter_style.py @@ -164,7 +164,10 @@ def azdev_on_external_extension(index_json): azdev_extension = AzdevExtensionHelper(name) azdev_extension.linter() - azdev_extension.style() + # TODO: + # azdev style support external extension + # azdev test support external extension + # azdev_extension.style() print('Checking service name for external extensions') service_name.check() From 9f9d59415424caf4e385c3d7252f745c7a116062 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 7 Dec 2023 11:47:49 +0800 Subject: [PATCH 24/33] Update __init__.py --- src/containerapp/azext_containerapp/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/containerapp/azext_containerapp/__init__.py b/src/containerapp/azext_containerapp/__init__.py index 45b0764676e..e0b1eccac7a 100644 --- a/src/containerapp/azext_containerapp/__init__.py +++ b/src/containerapp/azext_containerapp/__init__.py @@ -1,4 +1,3 @@ -#test # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. From 537f83682a8521d30fce46de84286adf059038f9 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 12 Dec 2023 16:37:46 +0800 Subject: [PATCH 25/33] todo --- scripts/ci/azdev_linter_style.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/ci/azdev_linter_style.py b/scripts/ci/azdev_linter_style.py index 21735461d16..4d30cba8221 100644 --- a/scripts/ci/azdev_linter_style.py +++ b/scripts/ci/azdev_linter_style.py @@ -202,6 +202,8 @@ def azdev_on_internal_extension(modified_files): def main(): + # TODO: + # Add parameters to control whether azdev linter, azdev style, azdev test needs to be run. modified_files = find_modified_files_against_master_branch() if len(modified_files) == 1 and contain_index_json(modified_files): From f05925dd98c8c4e57a266ff978a280c80488f12c Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 12 Dec 2023 17:44:20 +0800 Subject: [PATCH 26/33] update --- azure-pipelines.yml | 29 +++++++++++++++++++++++++++-- scripts/ci/azdev_linter_style.py | 28 +++++++++++++++++++--------- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5911c056653..8a97efeb854 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -117,6 +117,30 @@ jobs: ADO_PULL_REQUEST_LATEST_COMMIT: HEAD ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) +- job: AzdevStyleModifiedExtensions + displayName: "azdev style on Modified Extensions" + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + pool: + name: 'pool-ubuntu-2004' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: 3.11 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + #!/usr/bin/env bash + set -ev + source ./env/bin/activate + # overwrite the default AZURE_EXTENSION_DIR set by ADO + AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version + + AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/azdev_linter_style.py --type style + displayName: "azdev style on Modified Extensions" + env: + ADO_PULL_REQUEST_LATEST_COMMIT: HEAD + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) + - job: AzdevModifiedExtensions displayName: "azdev linter && style on Modified Extensions" condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) @@ -134,8 +158,9 @@ jobs: source ./env/bin/activate # overwrite the default AZURE_EXTENSION_DIR set by ADO AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version - - AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/azdev_linter_style.py + + # TODO: remove --type linter once all extensions are fixed + AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/azdev_linter_style.py --type linter displayName: "azdev linter && style on Modified Extensions" env: ADO_PULL_REQUEST_LATEST_COMMIT: HEAD diff --git a/scripts/ci/azdev_linter_style.py b/scripts/ci/azdev_linter_style.py index 4d30cba8221..e29e68a4397 100644 --- a/scripts/ci/azdev_linter_style.py +++ b/scripts/ci/azdev_linter_style.py @@ -134,7 +134,7 @@ def contain_extension_code(files): return False -def azdev_on_external_extension(index_json): +def azdev_on_external_extension(index_json, azdev_type): """ Check if the modified metadata items in index.json refer to the extension in repo. If not, az extension check on wheel. Otherwise skip it. @@ -163,7 +163,8 @@ def azdev_on_external_extension(index_json): az_extension.add_from_url(latest_entry['downloadUrl']) azdev_extension = AzdevExtensionHelper(name) - azdev_extension.linter() + if azdev_type in ['all', 'linter']: + azdev_extension.linter() # TODO: # azdev style support external extension # azdev test support external extension @@ -175,7 +176,7 @@ def azdev_on_external_extension(index_json): az_extension.remove() -def azdev_on_internal_extension(modified_files): +def azdev_on_internal_extension(modified_files, azdev_type): extension_names = set() for f in modified_files: @@ -192,8 +193,10 @@ def azdev_on_internal_extension(modified_files): azdev_extension = AzdevExtensionHelper(name) azdev_extension.add_from_code() - azdev_extension.linter() - azdev_extension.style() + if azdev_type in ['all', 'linter']: + azdev_extension.linter() + if azdev_type in ['all', 'style']: + azdev_extension.style() print('Checking service name for internal extensions') service_name.check() @@ -202,8 +205,15 @@ def azdev_on_internal_extension(modified_files): def main(): - # TODO: - # Add parameters to control whether azdev linter, azdev style, azdev test needs to be run. + import argparse + parser = argparse.ArgumentParser(description='azdev linter and azdev style on modified extensions') + parser.add_argument('--type', + type=str, + help='Control whether azdev linter, azdev style, azdev test needs to be run. ' + 'Supported values: linter, style, test, all, all is the default.', default='all') + args = parser.parse_args() + azdev_type = args.type + print('azdev type:', azdev_type) modified_files = find_modified_files_against_master_branch() if len(modified_files) == 1 and contain_index_json(modified_files): @@ -212,7 +222,7 @@ def main(): # If the modified metadata items refer to the extension code exits in this repo, PR is be created via Pipeline. # If the modified metadata items refer to the extension code doesn't exist, PR is created from Service Team. # We try to run azdev linter and azdev style on it. - azdev_on_external_extension(modified_files[0]) + azdev_on_external_extension(modified_files[0], azdev_type) else: # modified files contain more than one file @@ -221,7 +231,7 @@ def main(): if contain_index_json(modified_files): raise ModifiedFilesNotAllowedError() - azdev_on_internal_extension(modified_files) + azdev_on_internal_extension(modified_files, azdev_type) else: separator_line() print('no extension source code modified, no extension needs to be checked') From a373c6c203cb0d64fb5b4e1536a7f48477cc6f23 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 12 Dec 2023 17:46:34 +0800 Subject: [PATCH 27/33] update --- azure-pipelines.yml | 1 + src/containerapp/azext_containerapp/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8a97efeb854..1071a6d0295 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -120,6 +120,7 @@ jobs: - job: AzdevStyleModifiedExtensions displayName: "azdev style on Modified Extensions" condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + continueOnError: true pool: name: 'pool-ubuntu-2004' steps: diff --git a/src/containerapp/azext_containerapp/__init__.py b/src/containerapp/azext_containerapp/__init__.py index e0b1eccac7a..ab35c0e50f1 100644 --- a/src/containerapp/azext_containerapp/__init__.py +++ b/src/containerapp/azext_containerapp/__init__.py @@ -1,3 +1,4 @@ +# test # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. From e48d1a3682ec813a2831a4c70e44649433595c32 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 13 Dec 2023 10:53:33 +0800 Subject: [PATCH 28/33] Update __init__.py --- src/containerapp/azext_containerapp/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/containerapp/azext_containerapp/__init__.py b/src/containerapp/azext_containerapp/__init__.py index ab35c0e50f1..e0b1eccac7a 100644 --- a/src/containerapp/azext_containerapp/__init__.py +++ b/src/containerapp/azext_containerapp/__init__.py @@ -1,4 +1,3 @@ -# test # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. From c670b23f579662fb1b41f5f519f1c4ab9b864af2 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 13 Dec 2023 11:03:22 +0800 Subject: [PATCH 29/33] Update azure-pipelines.yml --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1071a6d0295..2b7b9b4e099 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -142,8 +142,8 @@ jobs: ADO_PULL_REQUEST_LATEST_COMMIT: HEAD ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) -- job: AzdevModifiedExtensions - displayName: "azdev linter && style on Modified Extensions" +- job: AzdevLinterModifiedExtensions + displayName: "azdev linter on Modified Extensions" condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) pool: name: 'pool-ubuntu-2004' @@ -162,7 +162,7 @@ jobs: # TODO: remove --type linter once all extensions are fixed AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/azdev_linter_style.py --type linter - displayName: "azdev linter && style on Modified Extensions" + displayName: "azdev linter on Modified Extensions" env: ADO_PULL_REQUEST_LATEST_COMMIT: HEAD ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) From 71b9b3ec57aa594c59d57a662d72bc565f617153 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 13 Dec 2023 11:15:10 +0800 Subject: [PATCH 30/33] update --- scripts/ci/azdev_linter_style.py | 33 ++++++++++--------- .../azext_containerapp/__init__.py | 1 + 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/scripts/ci/azdev_linter_style.py b/scripts/ci/azdev_linter_style.py index e29e68a4397..aee9633425e 100644 --- a/scripts/ci/azdev_linter_style.py +++ b/scripts/ci/azdev_linter_style.py @@ -9,17 +9,18 @@ It's only working on ADO by default. If want to run locally, please update the target branch/commit to find diff in function find_modified_files_against_master_branch() """ - -import os import json -from subprocess import check_output, check_call -from pkg_resources import parse_version +import logging +import os +from subprocess import check_call, check_output import service_name +from pkg_resources import parse_version +logger = logging.getLogger(__name__) def separator_line(): - print('-' * 100) + logger.info('-' * 100) class ModifiedFilesNotAllowedError(Exception): @@ -50,7 +51,7 @@ def __init__(self, extension_name): @staticmethod def _cmd(cmd): - print(cmd) + logger.info(cmd) check_call(cmd, shell=True) def add_from_url(self, url): @@ -66,7 +67,7 @@ def __init__(self, extension_name): @staticmethod def _cmd(cmd): - print(cmd) + logger.info(cmd) check_call(cmd, shell=True) def add_from_code(self): @@ -94,17 +95,17 @@ def find_modified_files_against_master_branch(): ado_pr_target_branch = 'origin/' + os.environ.get('ADO_PULL_REQUEST_TARGET_BRANCH') separator_line() - print('pull request target branch:', ado_pr_target_branch) + logger.info('pull request target branch:', ado_pr_target_branch) cmd = 'git --no-pager diff --name-only --diff-filter=ACMRT {} -- src/'.format(ado_pr_target_branch) files = check_output(cmd.split()).decode('utf-8').split('\n') files = [f for f in files if len(f) > 0] if files: - print('modified files:') + logger.info('modified files:') separator_line() for f in files: - print(f) + logger.info(f) return files @@ -170,7 +171,7 @@ def azdev_on_external_extension(index_json, azdev_type): # azdev test support external extension # azdev_extension.style() - print('Checking service name for external extensions') + logger.info('Checking service name for external extensions') service_name.check() az_extension.remove() @@ -186,7 +187,7 @@ def azdev_on_internal_extension(modified_files, azdev_type): if not extension_names: separator_line() - print('no extension source code modified, no extension needs to be checked') + logger.info('no extension source code modified, no extension needs to be checked') for name in extension_names: separator_line() @@ -198,7 +199,7 @@ def azdev_on_internal_extension(modified_files, azdev_type): if azdev_type in ['all', 'style']: azdev_extension.style() - print('Checking service name for internal extensions') + logger.info('Checking service name for internal extensions') service_name.check() azdev_extension.remove() @@ -213,7 +214,7 @@ def main(): 'Supported values: linter, style, test, all, all is the default.', default='all') args = parser.parse_args() azdev_type = args.type - print('azdev type:', azdev_type) + logger.info('azdev type:', azdev_type) modified_files = find_modified_files_against_master_branch() if len(modified_files) == 1 and contain_index_json(modified_files): @@ -234,7 +235,7 @@ def main(): azdev_on_internal_extension(modified_files, azdev_type) else: separator_line() - print('no extension source code modified, no extension needs to be checked') + logger.info('no extension source code modified, no extension needs to be checked') separator_line() @@ -242,5 +243,5 @@ def main(): try: main() except ModifiedFilesNotAllowedError as e: - print(e) + logger.error(e) exit(1) diff --git a/src/containerapp/azext_containerapp/__init__.py b/src/containerapp/azext_containerapp/__init__.py index e0b1eccac7a..ab35c0e50f1 100644 --- a/src/containerapp/azext_containerapp/__init__.py +++ b/src/containerapp/azext_containerapp/__init__.py @@ -1,3 +1,4 @@ +# test # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. From 8bd03194cd7c7fdb7fbec72e0d2c4fc6d98d558e Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 13 Dec 2023 11:23:50 +0800 Subject: [PATCH 31/33] update --- scripts/ci/azdev_linter_style.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/ci/azdev_linter_style.py b/scripts/ci/azdev_linter_style.py index aee9633425e..1c2c67c1775 100644 --- a/scripts/ci/azdev_linter_style.py +++ b/scripts/ci/azdev_linter_style.py @@ -18,6 +18,10 @@ from pkg_resources import parse_version logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +logger.addHandler(ch) def separator_line(): logger.info('-' * 100) From 0a0f70bb47a2a899a22ceae948c1a180475abe9d Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 13 Dec 2023 11:32:09 +0800 Subject: [PATCH 32/33] Update azdev_linter_style.py --- scripts/ci/azdev_linter_style.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/azdev_linter_style.py b/scripts/ci/azdev_linter_style.py index 1c2c67c1775..23a06eb7d67 100644 --- a/scripts/ci/azdev_linter_style.py +++ b/scripts/ci/azdev_linter_style.py @@ -99,7 +99,7 @@ def find_modified_files_against_master_branch(): ado_pr_target_branch = 'origin/' + os.environ.get('ADO_PULL_REQUEST_TARGET_BRANCH') separator_line() - logger.info('pull request target branch:', ado_pr_target_branch) + logger.info('pull request target branch: %s', ado_pr_target_branch) cmd = 'git --no-pager diff --name-only --diff-filter=ACMRT {} -- src/'.format(ado_pr_target_branch) files = check_output(cmd.split()).decode('utf-8').split('\n') @@ -218,7 +218,7 @@ def main(): 'Supported values: linter, style, test, all, all is the default.', default='all') args = parser.parse_args() azdev_type = args.type - logger.info('azdev type:', azdev_type) + logger.info('azdev type: %s', azdev_type) modified_files = find_modified_files_against_master_branch() if len(modified_files) == 1 and contain_index_json(modified_files): From 93c7c14d463b2f44e8d232973826bca1241189fc Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 13 Dec 2023 11:42:19 +0800 Subject: [PATCH 33/33] update --- scripts/ci/source_code_static_analysis.py | 148 ------------------ .../azext_containerapp/__init__.py | 1 - 2 files changed, 149 deletions(-) delete mode 100644 scripts/ci/source_code_static_analysis.py diff --git a/scripts/ci/source_code_static_analysis.py b/scripts/ci/source_code_static_analysis.py deleted file mode 100644 index 2247e551b9f..00000000000 --- a/scripts/ci/source_code_static_analysis.py +++ /dev/null @@ -1,148 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from __future__ import print_function - -import os -import sys -import multiprocessing - -from pylint import lint -from flake8.main import application - -from util import get_repo_root - -from verify_codeowners import main as _verify_codeowners - - -SDK_FILE_MARKER = r"# Code generated by Microsoft (R) AutoRest Code Generator." - - -# build list of sdk files -def _get_sdk_module_list(root_dir): - dir_paths = [os.path.join(root_dir, path) for path in os.listdir(root_dir)] - sdk_files = [] - - for path in dir_paths: - if os.path.isdir(path): - sdk_files.extend(_get_sdk_module_list(path)) - elif _is_sdk_file(path): - sdk_files.append(os.path.dirname(path)) - - return sdk_files - - -# check if the current file is a python sdk file -def _is_sdk_file(file_path): - # don't bother opening non-python files. e.g pyc files. - if not file_path.endswith(".py"): - return False - with open(file_path, encoding='utf-8') as f: - for line in f: - if SDK_FILE_MARKER in line: - return True - return False - - -# build list of modules that start with prefix "azext_" in the src -def _get_azext_module_paths(root_dir): - src_root = os.path.join(root_dir, "src") - src_paths = [os.path.join(src_root, path) for path in os.listdir(src_root)] - ext_modules = [] - for ext_root in src_paths: - if os.path.isdir(ext_root): - paths = [path for path in os.listdir(ext_root)] - ext_modules.extend([os.path.join(ext_root, path) for path in paths if path.startswith("azext_")]) - return ext_modules - - -# build list of python ci scripts -def _get_ci_py_file_paths(directory): - return [os.path.join(directory, path) for path in os.listdir(directory) if path.endswith(".py")] - - -def _run_pylint(module_paths, ignored_modules=None, rcfile=None, cpu_count=1): - pylint_opts = [] - - if ignored_modules: - pylint_opts.append("--ignore={}".format(ignored_modules)) - if rcfile: - pylint_opts.append("--rcfile={}".format(rcfile)) - - pylint_opts.append("--jobs={}".format(cpu_count)) - pylint_opts.extend(module_paths) - - try: - lint.Run(pylint_opts) - except SystemExit as se: - # 0: everything is fine - # 1: Fatal message issued - # 2: Error message issued - # 4: Warning message issued - # 8: Refactor message issued - # 16: Convention message issued - # 32: Usage error - if se.code != 0: - sys.exit(se.code) - - -def _run_flake8(module_paths, config_file=None): - flake8_opts = ["--statistics"] - - if config_file: - flake8_opts.append("--append-config={}".format(config_file)) - - flake8_opts.extend(module_paths) - - app = application.Application() - app.run(flake8_opts) - try: - app.exit() - except SystemExit: - pass - - if app.result_count > 0 or app.catastrophic_failure: - sys.exit(app.result_count or 1) - - -def main(): - cpu_count = multiprocessing.cpu_count() - - root_dir = get_repo_root() - sdk_modules = _get_sdk_module_list(root_dir) - sdk_modules.append("vendored_sdks") - module_paths = _get_azext_module_paths(root_dir) - - scripts_dir = os.path.join(root_dir, "scripts") - ci_files = _get_ci_py_file_paths(os.path.join(scripts_dir, "ci")) - - rc_file = os.path.join(root_dir, "pylintrc") - config_file = os.path.join(root_dir, ".flake8") - - print("\nRunning pylint on extensions...") - _run_pylint(module_paths, ",".join(sdk_modules), rc_file, cpu_count) - print("Pylint OK.\n") - - print("Running flake8 on extensions...") - _run_flake8(module_paths, config_file) - print("Flake8 OK.\n") - - print("Running pylint on CI scripts...") - _run_pylint(ci_files, rcfile=rc_file, cpu_count=cpu_count) - print("Pylint OK.\n") - - print("Running flake8 on CI scripts...") - _run_flake8(ci_files, config_file=config_file) - print("Pylint OK.\n") - - print("Other Static checks...") - - _verify_codeowners() - - print("All static checks successful!") - - -if __name__ == "__main__": - main() diff --git a/src/containerapp/azext_containerapp/__init__.py b/src/containerapp/azext_containerapp/__init__.py index ab35c0e50f1..e0b1eccac7a 100644 --- a/src/containerapp/azext_containerapp/__init__.py +++ b/src/containerapp/azext_containerapp/__init__.py @@ -1,4 +1,3 @@ -# test # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information.