[AutoPR containerregistry/resource-manager] ACR with Autorest for Python 4

.azure-pipelines/client.yml

@@ -10,7 +10,8 @@ variables:
   PythonVersion: '3.6'
   ReleaseTag: 'RELEASE_CANDIDATE'
   TestMarkArgument: 'not cosmosEmulator'
-
+  skipComponentGovernanceDetection: true
+
 jobs:
   - job: 'Build'
 

.azure-pipelines/docs.yml

@@ -6,6 +6,8 @@ variables:
 
 jobs:
   - job: 'DocGen'
+    variables:
+      skipComponentGovernanceDetection: true
     timeoutInMinutes: 120
     pool:
       vmImage: 'vs2017-win2016'

.azure-pipelines/tests-nightly-python.yml

@@ -4,6 +4,8 @@ trigger:
 jobs:
 
   - job: Validate_Nightly_Python_Build
+    variables:
+      skipComponentGovernanceDetection: true
 
     timeoutInMinutes: 90
 

.azure-pipelines/tests.yml

@@ -3,7 +3,9 @@
 
 jobs:
   - job: 'Test'
-
+    variables:
+      skipComponentGovernanceDetection: true
+
     timeoutInMinutes: 120
     strategy:
       matrix:

.azure-pipelines/update_pr.yml

@@ -6,6 +6,9 @@
 trigger:
 - master
 
+variables:
+  skipComponentGovernanceDetection: true
+
 pool:
   vmImage: 'ubuntu-latest'
 
@@ -15,7 +18,7 @@ steps:
   inputs:
     versionSpec: 3.7
 
-- script: python3 -m pip install -e "git+https://github.com/Azure/azure-sdk-for-python#subdirectory=azure-sdk-tools&egg=azure-sdk-tools"
+- script: python3 -m pip install -e "git+https://github.com/Azure/azure-sdk-for-python#subdirectory=tools/azure-sdk-tools&egg=azure-sdk-tools"
   displayName: 'Install Azure SDK tools'
 
 - script: python3  -m packaging_tools.update_pr -v --pr-number $(System.PullRequest.PullRequestNumber) --repo $(Build.Repository.Name)

.docsettings.yml

@@ -1,5 +1,5 @@
 omitted_paths:
-  - azure-sdk-tools/*
+  - tools/*
   - scripts/*
   - azure-mgmt-netapp/tests/*
   - "azure-mgmt*/*"

scripts/dev_setup.py

@@ -14,81 +14,118 @@
 from collections import Counter
 from subprocess import check_call, CalledProcessError
 
-root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..'))
+root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", ".."))
 
-def pip_command(command, additional_dir='.', error_ok=False):
+
+def pip_command(command, additional_dir=".", error_ok=False):
     try:
-        print('Executing: {} from {}'.format(command, additional_dir))
-        check_call([sys.executable, '-m', 'pip'] + command.split(), cwd=os.path.join(root_dir, additional_dir))
+        print("Executing: {} from {}".format(command, additional_dir))
+        check_call(
+            [sys.executable, "-m", "pip"] + command.split(),
+            cwd=os.path.join(root_dir, additional_dir),
+        )
         print()
     except CalledProcessError as err:
         print(err, file=sys.stderr)
         if not error_ok:
             sys.exit(1)
 
+
 # optional argument in a situation where we want to build a variable subset of packages
-parser = argparse.ArgumentParser(description='Set up the dev environment for selected packages.')
-parser.add_argument('--packageList', '-p',
-    dest='packageList',
-    default='',
-    help='Comma separated list of targeted packages. Used to limit the number of packages that dependencies will be installed for.')
+parser = argparse.ArgumentParser(
+    description="Set up the dev environment for selected packages."
+)
+parser.add_argument(
+    "--packageList",
+    "-p",
+    dest="packageList",
+    default="",
+    help="Comma separated list of targeted packages. Used to limit the number of packages that dependencies will be installed for.",
+)
 args = parser.parse_args()
 
-packages = {('.', os.path.dirname(p)) for p in glob.glob('azure*/setup.py')}
-# Handle the SDK folder as well
-packages.update({tuple(os.path.dirname(f).rsplit(os.sep, 1)) for f in glob.glob('sdk/*/azure*/setup.py')})
+packages = {
+    tuple(os.path.dirname(f).rsplit(os.sep, 1))
+    for f in glob.glob("sdk/*/azure*/setup.py") + glob.glob("tools/azure*/setup.py")
+}
 # [(base_folder, package_name), ...] to {package_name: base_folder, ...}
 packages = {package_name: base_folder for (base_folder, package_name) in packages}
 
 # keep targeted packages separate. python2 needs the nspkgs to work properly.
 if not args.packageList:
     targeted_packages = list(packages.keys())
 else:
-    targeted_packages = [os.path.relpath(x.strip()) for x in args.packageList.split(',')]
+    targeted_packages = [
+        os.path.relpath(x.strip()) for x in args.packageList.split(",")
+    ]
 
 # Extract nspkg and sort nspkg by number of "-"
-nspkg_packages = [p for p in packages.keys() if 'nspkg' in p]
-nspkg_packages.sort(key = lambda x: len([c for c in x if c == '-']))
+nspkg_packages = [p for p in packages.keys() if "nspkg" in p]
+nspkg_packages.sort(key=lambda x: len([c for c in x if c == "-"]))
 
 # Manually push meta-packages at the end, in reverse dependency order
-meta_packages = ['azure-mgmt', 'azure']
+meta_packages = ["azure-mgmt", "azure"]
 
-content_packages = sorted([p for p in packages.keys() if p not in nspkg_packages+meta_packages and p in targeted_packages])
+content_packages = sorted(
+    [
+        p
+        for p in packages.keys()
+        if p not in nspkg_packages + meta_packages and p in targeted_packages
+    ]
+)
 
-# Put azure-common in front
-if 'azure-common' in content_packages:
-    content_packages.remove('azure-common')
-content_packages.insert(0, 'azure-common')
+# Install tests dep first
+if "azure-devtools" in content_packages:
+    content_packages.remove("azure-devtools")
+content_packages.insert(0, "azure-devtools")
 
-if 'azure-sdk-tools' in content_packages:
-    content_packages.remove('azure-sdk-tools')
-content_packages.insert(1, 'azure-sdk-tools')
+if "azure-sdk-tools" in content_packages:
+    content_packages.remove("azure-sdk-tools")
+content_packages.insert(1, "azure-sdk-tools")
 
-print('Running dev setup...')
-print('Root directory \'{}\'\n'.format(root_dir))
+# Put azure-common in front of content package
+if "azure-common" in content_packages:
+    content_packages.remove("azure-common")
+content_packages.insert(2, "azure-common")
+
+print("Running dev setup...")
+print("Root directory '{}'\n".format(root_dir))
 
 # install private whls if there are any
-privates_dir = os.path.join(root_dir, 'privates')
+privates_dir = os.path.join(root_dir, "privates")
 if os.path.isdir(privates_dir) and os.listdir(privates_dir):
-    whl_list = ' '.join([os.path.join(privates_dir, f) for f in os.listdir(privates_dir)])
-    pip_command('install {}'.format(whl_list))
+    whl_list = " ".join(
+        [os.path.join(privates_dir, f) for f in os.listdir(privates_dir)]
+    )
+    pip_command("install {}".format(whl_list))
 
 # install nspkg only on py2, but in wheel mode (not editable mode)
-if sys.version_info < (3, ):
+if sys.version_info < (3,):
     for package_name in nspkg_packages:
-        pip_command('install {}/{}/'.format(packages[package_name], package_name))
+        pip_command("install {}/{}/".format(packages[package_name], package_name))
 
 # install packages
+print("Packages to install: {}".format(content_packages))
 for package_name in content_packages:
-    print("Installing {}".format(package_name))
+    print("\nInstalling {}".format(package_name))
     # if we are running dev_setup with no arguments. going after dev_requirements will be a pointless exercise
     # and waste of cycles as all the dependencies will be installed regardless.
-    if os.path.isfile('{}/{}/dev_requirements.txt'.format(packages[package_name], package_name)):
-        pip_command('install -r dev_requirements.txt', os.path.join(packages[package_name], package_name))
-    pip_command('install --ignore-requires-python -e {}'.format(os.path.join(packages[package_name], package_name)))
+    if os.path.isfile(
+        "{}/{}/dev_requirements.txt".format(packages[package_name], package_name)
+    ):
+        pip_command(
+            "install -r dev_requirements.txt",
+            os.path.join(packages[package_name], package_name),
+        )
+    pip_command(
+        "install --ignore-requires-python -e {}".format(
+            os.path.join(packages[package_name], package_name)
+        )
+    )
 
 # On Python 3, uninstall azure-nspkg if he got installed
-if sys.version_info >= (3, ):
-    pip_command('uninstall -y azure-nspkg', error_ok=True)
+if sys.version_info >= (3,):
+    pip_command("uninstall -y azure-nspkg", error_ok=True)
+
+print("Finished dev setup.")
 
-print('Finished dev setup.')

scripts/devops_tasks/common_tasks.py

@@ -33,10 +33,14 @@ def process_glob_string(glob_string, target_root_dir):
     # dedup, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*"
     return list(set(collected_top_level_directories))
 
-def run_check_call(command_array, working_directory, acceptable_return_codes = []):
-    print('Command Array: {0}, Target Working Directory: {1}'.format(command_array, working_directory))
+def run_check_call(command_array, working_directory, acceptable_return_codes = [], run_as_shell = False):
     try:
-        check_call(command_array, cwd = working_directory)
+        if run_as_shell:
+            print('Command Array: {0}, Target Working Directory: {1}'.format(' '.join(command_array), working_directory))
+            check_call(' '.join(command_array), cwd = working_directory, shell = True)
+        else:
+            print('Command Array: {0}, Target Working Directory: {1}'.format(command_array, working_directory))
+            check_call(command_array, cwd = working_directory)
     except CalledProcessError as err:
         if err.returncode not in acceptable_return_codes:
             print(err) #, file = sys.stderr

scripts/devops_tasks/setup_execute_tests.py

@@ -38,7 +38,7 @@ def prep_and_run_tests(targeted_packages, python_version, test_res):
     command_array = [python_version, '-m', 'pytest']
     command_array.extend(test_res)
     command_array.extend(targeted_packages)
-    run_check_call(command_array, root_dir, ALLOWED_RETURN_CODES)
+    run_check_call(command_array, root_dir, ALLOWED_RETURN_CODES, True)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description = 'Install Dependencies, Install Packages, Test Azure Packages, Called from DevOps YAML Pipeline')