diff --git a/tasks.py b/tasks.py index 09fd7a26942..3de7c3dc091 100644 --- a/tasks.py +++ b/tasks.py @@ -5,14 +5,18 @@ # -------------------------------------------------------------------------- from multiprocessing import Pool import os -import shutil +from typing import Any, Dict, Optional +from enum import Enum, auto from colorama import init, Fore from invoke import task, run init() -_AUTOREST_CMD_LINE = "autorest" +class _SwaggerGroup(str, Enum): + VANILLA = auto() + AZURE = auto() + AZURE_ARM = auto() -service_to_readme_path = { +_SERVICE_TO_README_PATH = { 'azure-ai-textanalytics': 'test/services/azure-ai-textanalytics/README.md', 'azure-ai-formrecognizer': 'test/services/azure-ai-formrecognizer/README.md', 'azure-storage-blob': '../azure-sdk-for-python/sdk/storage/azure-storage-blob/swagger/README.md', @@ -24,289 +28,234 @@ 'azure-keyvault': 'test/services/azure-keyvault/README.md', } -default_mappings = { - 'AcceptanceTests/AdditionalProperties': 'additionalProperties.json', - 'AcceptanceTests/ParameterFlattening': 'parameter-flattening.json', - 'AcceptanceTests/BodyArray': 'body-array.json', - 'AcceptanceTests/BodyBoolean': 'body-boolean.json', - 'AcceptanceTests/BodyByte': 'body-byte.json', - 'AcceptanceTests/BodyComplex': 'body-complex.json', - 'AcceptanceTests/BodyDate': 'body-date.json', - 'AcceptanceTests/BodyDateTime': 'body-datetime.json', - 'AcceptanceTests/BodyDateTimeRfc1123': 'body-datetime-rfc1123.json', - 'AcceptanceTests/BodyDuration': 'body-duration.json', - 'AcceptanceTests/BodyDictionary': 'body-dictionary.json', - 'AcceptanceTests/BodyFile': 'body-file.json', - 'AcceptanceTests/Constants': 'constants.json', - 'AcceptanceTests/BodyFormData': 'body-formdata.json', - 'AcceptanceTests/BodyInteger': 'body-integer.json', - 'AcceptanceTests/BodyNumber': 'body-number.json', - 'AcceptanceTests/BodyString': 'body-string.json', - 'AcceptanceTests/BodyTime': 'body-time.json', - 'AcceptanceTests/ExtensibleEnums': ['extensible-enums-swagger.json', 'extensibleenumsswagger'], - 'AcceptanceTests/Header': 'header.json', - 'AcceptanceTests/Http': ['httpInfrastructure.json', 'httpinfrastructure'], - 'AcceptanceTests/Report': 'report.json', - 'AcceptanceTests/RequiredOptional': 'required-optional.json', - 'AcceptanceTests/Url': 'url.json', - 'AcceptanceTests/Validation': 'validation.json', - 'AcceptanceTests/CustomBaseUri': ['custom-baseUrl.json', 'custombaseurl'], - 'AcceptanceTests/CustomBaseUriMoreOptions': ['custom-baseUrl-more-options.json', 'custombaseurlmoreoptions'], - 'AcceptanceTests/ModelFlattening': 'model-flattening.json', - 'AcceptanceTests/Xml': ['xml-service.json', 'xmlservice'], - 'AcceptanceTests/UrlMultiCollectionFormat' : 'url-multi-collectionFormat.json', - 'AcceptanceTests/XmsErrorResponse': 'xms-error-responses.json', - 'AcceptanceTests/MediaTypes': 'media_types.json', - 'AcceptanceTests/ObjectType': 'object-type.json', - 'AcceptanceTests/NonStringEnums': 'non-string-enum.json', - 'AcceptanceTests/MultipleInheritance': 'multiple-inheritance.json', - 'AcceptanceTests/NoOperations': 'no-operations.json', +_VANILLA_SWAGGER_MAPPINGS = { + 'AdditionalProperties': 'additionalProperties.json', + 'ParameterFlattening': 'parameter-flattening.json', + 'BodyArray': 'body-array.json', + 'BodyBoolean': 'body-boolean.json', + 'BodyByte': 'body-byte.json', + 'BodyComplex': 'body-complex.json', + 'BodyDate': 'body-date.json', + 'BodyDateTime': 'body-datetime.json', + 'BodyDateTimeRfc1123': 'body-datetime-rfc1123.json', + 'BodyDuration': 'body-duration.json', + 'BodyDictionary': 'body-dictionary.json', + 'BodyFile': 'body-file.json', + 'Constants': 'constants.json', + 'BodyFormData': 'body-formdata.json', + 'BodyInteger': 'body-integer.json', + 'BodyNumber': 'body-number.json', + 'BodyString': 'body-string.json', + 'BodyTime': 'body-time.json', + 'ExtensibleEnums': 'extensible-enums-swagger.json', + 'Header': 'header.json', + 'Http': 'httpInfrastructure.json', + 'Report': 'report.json', + 'RequiredOptional': 'required-optional.json', + 'Url': 'url.json', + 'Validation': 'validation.json', + 'CustomBaseUri': 'custom-baseUrl.json', + 'CustomBaseUriMoreOptions': 'custom-baseUrl-more-options.json', + 'ModelFlattening': 'model-flattening.json', + 'Xml': 'xml-service.json', + 'UrlMultiCollectionFormat' : 'url-multi-collectionFormat.json', + 'XmsErrorResponse': 'xms-error-responses.json', + 'MediaTypes': 'media_types.json', + 'ObjectType': 'object-type.json', + 'NonStringEnums': 'non-string-enum.json', + 'MultipleInheritance': 'multiple-inheritance.json', + 'NoOperations': 'no-operations.json', } -default_azure_mappings = { - 'AcceptanceTests/AzureBodyDuration': ['body-duration.json', 'bodyduration'], - 'AcceptanceTests/AzureReport': 'azure-report.json', - 'AcceptanceTests/AzureParameterGrouping': 'azure-parameter-grouping.json', - 'AcceptanceTests/CustomBaseUri': ['custom-baseUrl.json', 'custombaseurl'], - 'AcceptanceTests/LroWithParameterizedEndpoints': 'lro-parameterized-endpoints.json', +_AZURE_SWAGGER_MAPPINGS = { + 'AzureBodyDuration': 'body-duration.json', + 'AzureReport': 'azure-report.json', + 'AzureParameterGrouping': 'azure-parameter-grouping.json', + 'CustomBaseUri': 'custom-baseUrl.json', + 'LroWithParameterizedEndpoints': 'lro-parameterized-endpoints.json', } # The list is mostly built on Swaggers that uses CloudError feature # These Swagger should be modified to test their features, and not the CloudError one -default_arm_mappings = { - 'AcceptanceTests/Head': 'head.json', - 'AcceptanceTests/HeadExceptions': 'head-exceptions.json', - 'AcceptanceTests/StorageManagementClient': ['storage.json', 'storage'], - 'AcceptanceTests/Lro': 'lro.json', - 'AcceptanceTests/SubscriptionIdApiVersion': 'subscriptionId-apiVersion.json', - 'AcceptanceTests/Paging': 'paging.json', - 'AcceptanceTests/CustomUrlPaging': ['custom-baseUrl-paging.json', 'custombaseurlpaging'], - 'AcceptanceTests/AzureSpecials': ['azure-special-properties.json', 'azurespecialproperties'], +_AZURE_ARM_SWAGGER_MAPPINGS = { + 'Head': 'head.json', + 'HeadExceptions': 'head-exceptions.json', + 'StorageManagementClient': 'storage.json', + 'Lro': 'lro.json', + 'SubscriptionIdApiVersion': 'subscriptionId-apiVersion.json', + 'Paging': 'paging.json', + 'CustomUrlPaging': 'custom-baseUrl-paging.json', + 'AzureSpecials': 'azure-special-properties.json', } -packages_with_client_side_validation = [ - 'AcceptanceTests/Validation', - 'AcceptanceTests/Url', - 'AcceptanceTests/RequiredOptional', - 'AcceptanceTests/CustomBaseUri', - 'AcceptanceTests/BodyComplex', - 'AcceptanceTests/AzureParameterGrouping', - 'AcceptanceTests/AzureSpecials' -] - -base_dir = os.path.dirname(__file__) - -swagger_dir = "node_modules/@microsoft.azure/autorest.testserver/swagger" - - -@task -def regen_expected(c, opts, debug): - output_dir = "{}/{}".format(opts['output_base_dir'], opts['output_dir']) if opts.get('output_base_dir') else opts['output_dir'] - keys = opts['mappings'].keys() +"""Overwrite default behavior we have assigned to test flags +""" + +_OVERWRITE_DEFAULT_NAMESPACE = { + 'ExtensibleEnums': 'extensibleenumsswagger', + 'Http': 'httpinfrastructure', + 'CustomBaseUri': 'custombaseurl', + 'CustomBaseUriMoreOptions': 'custombaseurlmoreoptions', + 'Xml': 'xmlservice', + 'AzureBodyDuration': 'bodyduration', + 'CustomUrlPaging': 'custombaseurlpaging', + 'AzureSpecials': 'azurespecialproperties', + 'StorageManagementClient': 'storage', +} - cmds = [] - for key in keys: - opts_mappings_value = opts['mappings'][key] - key = key.strip() - - swagger_files = (opts_mappings_value[0] if isinstance(opts_mappings_value, list) else opts_mappings_value).split(';') - output_folder = f"{output_dir}/{key}" - args = [ - f"--use={base_dir}", - # "--{}".format(opts['language']), - "--clear-output-folder", - f"--output-folder={output_folder}", - "--license-header={}".format(opts['header'] if opts.get('header') else 'MICROSOFT_MIT_NO_VERSION'), - "--enable-xml", - "--basic-setup-py", - "--package-version=0.1.0", - "--trace", - "--output-artifact=code-model-v4-no-tags", - ] - - for swagger_file in swagger_files: - input_file_name = "{}/{}".format(opts['input_base_dir'], swagger_file) if opts.get('input_base_dir') else swagger_file - args.append(f"--input-file={input_file_name}") +_PACKAGES_WITH_CLIENT_SIDE_VALIDATION = [ + 'Validation', + 'Url', + 'RequiredOptional', + 'CustomBaseUri', + 'BodyComplex', + 'AzureParameterGrouping', + 'AzureSpecials' +] - if debug: - args.append("--debug") - - if opts.get('add_credentials') and opts['add_credentials']: - args.append("--add-credentials=true") - - if opts.get('vanilla') and opts['vanilla']: - args.append("--vanilla=true") - - if opts.get('azure_arm') and opts['azure_arm']: - args.append("--azure-arm=true") - - if opts.get('flattening_threshold'): - args.append(f"--payload-flattening-threshold={opts['flattening_threshold']}") - - if opts.get('keep_version') and opts['keep_version']: - args.append("--keep-version-file=true") - - if opts.get('ns_prefix'): - if isinstance(opts_mappings_value, list) and len(opts_mappings_value) > 1 and isinstance(opts_mappings_value[1], str): - args.append(f"--namespace={opts_mappings_value[1]}") - else: - namespace = key.split('/')[-1].lower() - args.append(f"--namespace={namespace}") - - if opts.get('override-info.version'): - args.append(f"--override-info.version={opts['override-info.version']}") - if opts.get('override-info.title'): - args.append(f"--override-info.title={opts['override-info.title']}") - if opts.get('override-info.description'): - args.append(f"--override-info.description={opts['override-info.description']}") - if opts.get('credential-default-policy-type'): - args.append(f"--credential-default-policy-type={opts['credential-default-policy-type']}") - if opts.get('credential-key-header-name'): - args.append(f"--credential-key-header-name={opts['credential-key-header-name']}") - if opts.get('package-name'): - args.append(f"--package-name={opts['package-name']}") - if opts.get('override-client-name'): - args.append(f"--override-client-name={opts['override-client-name']}") - if key in packages_with_client_side_validation: - args.append("--client-side-validation=true") - - cmd_line = '{} {}'.format(_AUTOREST_CMD_LINE, " ".join(args)) - print(Fore.YELLOW + f'Queuing up: {cmd_line}') - cmds.append(cmd_line) +def _build_flags( + package_name: str, + swagger_name: str, + debug: bool, + swagger_group: _SwaggerGroup, + override_flags: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + autorest_dir = os.path.dirname(__file__) + testserver_dir = "node_modules/@microsoft.azure/autorest.testserver/swagger" + + if swagger_group == _SwaggerGroup.VANILLA: + generation_section = "vanilla" + else: + generation_section = "azure" + + flags = { + "use": autorest_dir, + "clear-output-folder": True, + "output-folder": f"test/{generation_section}/Expected/AcceptanceTests/{package_name}", + "license-header": "MICROSOFT_MIT_NO_VERSION", + "enable-xml": True, + "basic-setup-py": True, + "package-version": "0.1.0", + "trace": True, + "output-artifact": "code-model-v4-no-tags", + "input-file": f"{testserver_dir}/{swagger_name}", + "debug": debug, + "add-credential": False, + "vanilla": swagger_group == _SwaggerGroup.VANILLA, + "azure-arm": swagger_group == _SwaggerGroup.AZURE_ARM, + "payload-flattening-threshold": 1, + "keep-version-file": True, + "namespace": _OVERWRITE_DEFAULT_NAMESPACE.get(package_name, package_name.lower()), + "client-side-validation": package_name in _PACKAGES_WITH_CLIENT_SIDE_VALIDATION + } + if override_flags: + flags.update(override_flags) + return flags + +def _build_command_line( + package_name: str, + swagger_name: str, + debug: bool, + swagger_group: _SwaggerGroup, + override_flags: Optional[Dict[str, Any]] = None, +) -> str: + flags = _build_flags(package_name, swagger_name, debug, swagger_group, override_flags) + flag_strings = [ + f"--{flag}={value}" for flag, value in flags.items() + ] + return "autorest " + " ".join(flag_strings) +def _run_autorest(cmds, debug): if len(cmds) == 1: - success = run_autorest(cmds[0], debug=debug) + success = _run_single_autorest(cmds[0], debug=debug) else: # Execute actual taks in parallel with Pool() as pool: - result = pool.map(run_autorest, cmds) + result = pool.map(_run_single_autorest, cmds) success = all(result) if not success: - # Delete the old code, we can catch when it's not generating - shutil.rmtree(output_folder, ignore_errors=True) raise SystemExit("Autorest generation fails") -def run_autorest(cmd_line, debug=False): +def _run_single_autorest(cmd_line, debug=False): result = run(cmd_line, warn=True, hide=not debug) if result.ok or result.return_code is None: print(Fore.GREEN + f'Call "{cmd_line}" done with success') return True - else: - print(Fore.RED + f'Call "{cmd_line}" failed with {result.return_code}\n{result.stdout}\n{result.stderr}') - return False + print(Fore.RED + f'Call "{cmd_line}" failed with {result.return_code}\n{result.stdout}\n{result.stderr}') + return False + +def _regenerate( + mapping: Dict[str, str], + debug: bool, + swagger_group: _SwaggerGroup, + override_flags: Optional[Dict[str, Any]] = None, +) -> None: + cmds = [] + for package_name, swagger_name in mapping.items(): + command_line = _build_command_line(package_name, swagger_name, debug, swagger_group, override_flags) + print(Fore.YELLOW + f'Queuing up: {command_line}') + cmds.append(command_line) + _run_autorest(cmds, debug=debug) @task -def regenerate_python(c, swagger_name=None, debug=False): +def regenerate_vanilla(c, swagger_name=None, debug=False): if swagger_name: - default_mapping = {k: v for k, v in default_mappings.items() if swagger_name.lower() in k.lower()} + mapping = {k: v for k, v in _VANILLA_SWAGGER_MAPPINGS.items() if swagger_name.lower() in k.lower()} else: - default_mapping = default_mappings - opts = { - 'output_base_dir': 'test/vanilla', - 'input_base_dir': swagger_dir, - 'mappings': default_mapping, - 'output_dir': 'Expected', - 'flattening_threshold': '1', - 'vanilla': True, - 'keep_version': True, - 'ns_prefix': True - } - regen_expected(c, opts, debug) - + mapping = _VANILLA_SWAGGER_MAPPINGS + _regenerate(mapping, debug, swagger_group=_SwaggerGroup.VANILLA) @task -def regenerate_python_azure(c, swagger_name=None, debug=False): +def regenerate_azure(c, swagger_name=None, debug=False): if swagger_name: - default_mapping = {k: v for k, v in default_azure_mappings.items() if swagger_name.lower() in k.lower()} + mapping = {k: v for k, v in _AZURE_SWAGGER_MAPPINGS.items() if swagger_name.lower() in k.lower()} else: - default_mapping = default_azure_mappings - opts = { - 'output_base_dir': 'test/azure', - 'input_base_dir': swagger_dir, - 'mappings': default_mapping, - 'output_dir': 'Expected', - 'flattening_threshold': '1', - 'ns_prefix': True - } - regen_expected(c, opts, debug) - + mapping = _AZURE_SWAGGER_MAPPINGS + _regenerate(mapping, debug, swagger_group=_SwaggerGroup.AZURE) @task -def regenerate_python_arm(c, swagger_name=None, debug=False): +def regenerate_azure_arm(c, swagger_name=None, debug=False): if swagger_name: - default_mapping = {k: v for k, v in default_arm_mappings.items() if swagger_name.lower() in k.lower()} + mapping = {k: v for k, v in _AZURE_ARM_SWAGGER_MAPPINGS.items() if swagger_name.lower() in k.lower()} else: - default_mapping = default_arm_mappings - opts = { - 'output_base_dir': 'test/azure', - 'input_base_dir': swagger_dir, - 'mappings': default_mapping, - 'output_dir': 'Expected', - 'azure_arm': True, - 'flattening_threshold': '1', - 'ns_prefix': True - } - regen_expected(c, opts, debug) + mapping = _AZURE_ARM_SWAGGER_MAPPINGS + _regenerate(mapping, debug, swagger_group=_SwaggerGroup.AZURE_ARM) @task def regenerate_namespace_folders_test(c, debug=False): # regenerate a swagger (randomly chose BodyArray) to have a namespace length > 1 # to test pkgutil logic - default_mapping = {'AcceptanceTests/BodyArrayWithNamespaceFolders': ['body-array.json', 'vanilla.body.array']} - opts = { - 'output_base_dir': 'test/vanilla', - 'input_base_dir': swagger_dir, - 'mappings': default_mapping, - 'output_dir': 'Expected', - 'flattening_threshold': '1', - 'vanilla': True, - 'keep_version': True, - 'ns_prefix': True - } - regen_expected(c, opts, debug) + mapping = {'BodyArrayWithNamespaceFolders': 'body-array.json'} + override_flags = {"namespace": "vanilla.body.array"} + _regenerate(mapping, debug, swagger_group=_SwaggerGroup.VANILLA, override_flags=override_flags) @task def regenerate_credential_default_policy(c, debug=False): - default_mapping = {'AcceptanceTests/HeadWithAzureKeyCredentialPolicy': 'head.json'} - opts = { - 'output_base_dir': 'test/azure', - 'input_base_dir': swagger_dir, - 'mappings': default_mapping, - 'output_dir': 'Expected', - 'azure_arm': True, - 'flattening_threshold': '1', - 'ns_prefix': True, - 'credential-default-policy-type': 'AzureKeyCredentialPolicy', - 'credential-key-header-name': 'Authorization' + mapping = {'HeadWithAzureKeyCredentialPolicy': 'head.json'} + override_flags = { + "credential-default-policy-type": "AzureKeyCredentialPolicy", + "credential-key-header-name": "Authorization" } - regen_expected(c, opts, debug) + _regenerate(mapping, debug, swagger_group=_SwaggerGroup.AZURE_ARM, override_flags=override_flags) @task def regenerate_package_name_setup_py(c, debug=False): - default_mapping = {'AcceptanceTests/BodyByteWithPackageName': 'body-byte.json'} - opts = { - 'output_base_dir': 'test/vanilla', - 'input_base_dir': swagger_dir, - 'mappings': default_mapping, - 'output_dir': 'Expected', - 'flattening_threshold': '1', - 'vanilla': True, - 'keep_version': True, - 'ns_prefix': True, - 'package-name': 'package-name', - 'override-client-name': 'class_name' + mapping = {'BodyByteWithPackageName': 'body-byte.json'} + override_flags = { + "package-name": "package-name", + "override-client-name": "class_name" } - regen_expected(c, opts, debug) + _regenerate(mapping, debug, swagger_group=_SwaggerGroup.VANILLA, override_flags=override_flags) @task def regenerate(c, swagger_name=None, debug=False): # regenerate expected code for tests - regenerate_python(c, swagger_name, debug) - regenerate_python_azure(c, swagger_name, debug) - regenerate_python_arm(c, swagger_name, debug) + regenerate_vanilla(c, swagger_name, debug) + regenerate_azure(c, swagger_name, debug) + regenerate_azure_arm(c, swagger_name, debug) if not swagger_name: regenerate_namespace_folders_test(c, debug) regenerate_multiapi(c, debug) @@ -318,6 +267,7 @@ def regenerate(c, swagger_name=None, debug=False): @task def test(c, env=None): # run language-specific tests + base_dir = os.path.dirname(__file__) cmd = f'tox -e {env}' if env else 'tox' os.chdir(f"{base_dir}/test/vanilla/") c.run(cmd) @@ -329,35 +279,27 @@ def test(c, env=None): def regenerate_services(c, swagger_name=None, debug=False): # regenerate service from swagger if swagger_name: - service_mapping = {k: v for k, v in service_to_readme_path.items() if swagger_name.lower() in k.lower()} + service_mapping = {k: v for k, v in _SERVICE_TO_README_PATH.items() if swagger_name.lower() in k.lower()} else: - service_mapping = service_to_readme_path + service_mapping = _SERVICE_TO_README_PATH cmds = [] for service in service_mapping: - readme_path = service_to_readme_path[service] + readme_path = _SERVICE_TO_README_PATH[service] service = service.strip() - cmd_line = f'{_AUTOREST_CMD_LINE} {readme_path} --use=. --output-artifact=code-model-v4-no-tags' + cmd_line = f'autorest {readme_path} --use=. --output-artifact=code-model-v4-no-tags' if debug: cmd_line += " --debug" print(Fore.YELLOW + f'Queuing up: {cmd_line}') cmds.append(cmd_line) - if len(cmds) == 1: - success = run_autorest(cmds[0], debug=debug) - else: - # Execute actual taks in parallel - with Pool() as pool: - result = pool.map(run_autorest, cmds) - success = all(result) + _run_autorest(cmds[0], debug) - if not success: - raise SystemExit("Autorest generation fails") def _multiapi_command_line(location): cwd = os.getcwd() return ( - f'{_AUTOREST_CMD_LINE} {location} --use=. --multiapi --output-artifact=code-model-v4-no-tags ' + + f'autorest {location} --use=. --multiapi --output-artifact=code-model-v4-no-tags ' + f'--python-sdks-folder={cwd}/test/' ) @@ -381,18 +323,12 @@ def regenerate_multiapi(c, debug=False, swagger_name="test"): cmds = [_multiapi_command_line(spec) for spec in available_specifications if swagger_name.lower() in spec] - if len(cmds) == 1: - success = run_autorest(cmds[0], debug=debug) - else: - # Execute actual taks in parallel - with Pool() as pool: - result = pool.map(run_autorest, cmds) - success = all(result) + _run_autorest(cmds, debug) @task def regenerate_custom_poller_pager(c, debug=False): cwd = os.getcwd() cmd = ( - f'{_AUTOREST_CMD_LINE} test/azure/specification/custompollerpager/README.md --use=. --python-sdks-folder={cwd}/test/' + f'autorest test/azure/specification/custompollerpager/README.md --use=. --python-sdks-folder={cwd}/test/' ) - success = run_autorest(cmd, debug=debug) + _run_autorest([cmd], debug=debug)