diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index 26d69af37ca..7234b1afad6 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -3,37 +3,34 @@ SetLocal EnableDelayedExpansion echo build a msi installer using local cli sources and python executables. You need to have curl.exe, unzip.exe and msbuild.exe available under PATH echo. -set "PATH=%PATH%;%ProgramFiles%\Git\bin;%ProgramFiles%\Git\usr\bin;C:\Program Files (x86)\Git\bin;C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin" - -set PYTHON_DOWNLOAD_URL="https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-win32.zip" -set GET_PIP_DOWNLOAD_URL="https://bootstrap.pypa.io/get-pip.py" -set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" +set "PATH=%PATH%;%ProgramFiles%\Git\bin;%ProgramFiles%\Git\usr\bin;C:\Program Files (x86)\Git\bin;C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin;" if "%CLI_VERSION%"=="" ( echo Please set the CLI_VERSION environment variable, e.g. 2.0.13 goto ERROR ) +set PYTHON_VERSION=3.6.6 +set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" +set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python366-32.zip" :: Set up the output directory and temp. directories echo Cleaning previous build artifacts... - set OUTPUT_DIR=%~dp0..\out if exist %OUTPUT_DIR% rmdir /s /q %OUTPUT_DIR% mkdir %OUTPUT_DIR% set ARTIFACTS_DIR=%~dp0..\artifacts mkdir %ARTIFACTS_DIR% - set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch set BUILDING_DIR=%ARTIFACTS_DIR%\cli -set PYTHON_DIR=%ARTIFACTS_DIR%\Python set WIX_DIR=%ARTIFACTS_DIR%\wix +set PYTHON_DIR=%ARTIFACTS_DIR%\Python366-32 + set REPO_ROOT=%~dp0..\..\.. ::reset working folders if exist %BUILDING_DIR% rmdir /s /q %BUILDING_DIR% - ::rmdir always returns 0, so check folder's existence if exist %BUILDING_DIR% ( echo Failed to delete %BUILDING_DIR%. @@ -41,36 +38,6 @@ if exist %BUILDING_DIR% ( ) mkdir %BUILDING_DIR% -:: get Python -if exist %PYTHON_DIR% rmdir /s /q %PYTHON_DIR% - -:: rmdir always returns 0, so check folder's existence -if exist %PYTHON_DIR% ( - echo Failed to delete %PYTHON_DIR%. - goto ERROR -) - -mkdir %PYTHON_DIR% -pushd %PYTHON_DIR% - -echo Downloading Python. -curl -o python-archive.zip %PYTHON_DOWNLOAD_URL% -k -unzip -q python-archive.zip -unzip -q python36.zip -if %errorlevel% neq 0 goto ERROR - -del python-archive.zip -del python36.zip - -echo Python downloaded and extracted successfully. -echo Setting up pip -curl -o get-pip.py %GET_PIP_DOWNLOAD_URL% -k -%PYTHON_DIR%\python.exe get-pip.py -del get-pip.py -echo Pip set up successful. - -popd - if exist %TEMP_SCRATCH_FOLDER% rmdir /s /q %TEMP_SCRATCH_FOLDER% if exist %TEMP_SCRATCH_FOLDER% ( echo Failed to delete %TEMP_SCRATCH_FOLDER%. @@ -82,11 +49,10 @@ if exist %REPO_ROOT%\privates ( copy %REPO_ROOT%\privates\*.whl %TEMP_SCRATCH_FOLDER% ) -:: ensure wix is available +::ensure wix is available if exist %WIX_DIR% ( echo Using existing Wix at %WIX_DIR% ) - if not exist %WIX_DIR% ( mkdir %WIX_DIR% pushd %WIX_DIR% @@ -99,23 +65,34 @@ if not exist %WIX_DIR% ( popd ) -:: Use the Python version on the machine that creates the MSI +::ensure Python is available +if exist %PYTHON_DIR% ( + echo Using existing Python at %PYTHON_DIR% +) +if not exist %PYTHON_DIR% ( + mkdir %PYTHON_DIR% + pushd %PYTHON_DIR% + echo Downloading Python. + curl -o Python366-32.zip %PYTHON_DOWNLOAD_URL% -k + unzip -q Python366-32.zip + if %errorlevel% neq 0 goto ERROR + del Python366-32.zip + echo Python downloaded and extracted successfully. + popd +) +set PYTHON_EXE=%PYTHON_DIR%\python.exe + robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL :: Build & install all the packages with bdist_wheel +%BUILDING_DIR%\python.exe -m pip install wheel echo Building CLI packages... - -:: Workaround for get bdist_wheel to complete otherwise it fails to import azure_bdist_wheel -set PYTHONPATH=%BUILDING_DIR%\Lib\site-packages -del %BUILDING_DIR%\python36._pth - set CLI_SRC=%REPO_ROOT%\src for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nspkg %CLI_SRC%\azure-cli-telemetry) do ( pushd %%a %BUILDING_DIR%\python.exe setup.py bdist_wheel -d %TEMP_SCRATCH_FOLDER% popd ) - pushd %CLI_SRC%\command_modules for /D %%a in (*) do ( pushd %CLI_SRC%\command_modules\%%a @@ -123,16 +100,6 @@ for /D %%a in (*) do ( popd ) popd - -:: Undo the rest of the workaround and add site-packages to ._pth. -:: See https://docs.python.org/3/using/windows.html#finding-modules -set PYTHONPATH= -( - echo python36.zip - echo . - echo Lib\site-packages -) > %BUILDING_DIR%\python36._pth - echo Built CLI packages successfully. if %errorlevel% neq 0 goto ERROR @@ -142,10 +109,13 @@ for %%i in (%TEMP_SCRATCH_FOLDER%\*.whl) do ( set ALL_MODULES=!ALL_MODULES! %%i ) echo All modules: %ALL_MODULES% -%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall pycparser==2.18 %BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --no-cache-dir %ALL_MODULES% %BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg +pushd %BUILDING_DIR% +%BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py +popd + echo Creating the wbin (Windows binaries) folder that will be added to the path... mkdir %BUILDING_DIR%\wbin copy %REPO_ROOT%\build_scripts\windows\scripts\az.cmd %BUILDING_DIR%\wbin\ @@ -153,12 +123,6 @@ if %errorlevel% neq 0 goto ERROR copy %REPO_ROOT%\build_scripts\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR% copy %REPO_ROOT%\build_scripts\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR% -: Delete some files we don't need -rmdir /s /q %BUILDING_DIR%\Scripts -for /f %%a in ('dir %BUILDING_DIR%\Lib\site-packages\*.egg-info /b /s /a:d') do ( - rmdir /s /q %%a -) - :: Use universal files and remove Py3 only files pushd %BUILDING_DIR%\Lib\site-packages\azure\mgmt for /f %%a in ('dir /b /s *_py3.py') do ( @@ -172,11 +136,11 @@ for /f %%a in ('dir /b /s *_py3.*.pyc') do ( popd :: Remove .py and only deploy .pyc files -pushd %BUILDING_DIR%\Lib\site-packages\azure +pushd %BUILDING_DIR%\Lib\site-packages for /f %%f in ('dir /b /s *.pyc') do ( set PARENT_DIR=%%~df%%~pf.. - echo !PARENT_DIR! | findstr /C:"!BUILDING_DIR!\Lib\site-packages\pip" 1>nul - if errorlevel 1 ( + echo !PARENT_DIR! | findstr /C:\Lib\site-packages\pip\ 1>nul + if !errorlevel! neq 0 ( set FILENAME=%%~nf set BASE_FILENAME=!FILENAME:~0,-11! set pyc=!BASE_FILENAME!.pyc @@ -196,7 +160,7 @@ for /d /r %BUILDING_DIR%\Lib\site-packages\pip %%d in (__pycache__) do ( if %errorlevel% neq 0 goto ERROR echo Building MSI... -MSBuild.exe /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj +msbuild /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj start %OUTPUT_DIR% @@ -208,4 +172,4 @@ exit /b 1 :END exit /b 0 -popd +popd \ No newline at end of file diff --git a/build_scripts/windows/scripts/patch_models_v2.py b/build_scripts/windows/scripts/patch_models_v2.py new file mode 100644 index 00000000000..ce09f604385 --- /dev/null +++ b/build_scripts/windows/scripts/patch_models_v2.py @@ -0,0 +1,282 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from enum import Enum +import importlib +import inspect +import logging +from pathlib import Path +import pkgutil +import shutil +import sys +import tempfile + +from msrest.serialization import Model +from msrest.exceptions import HttpOperationError +from msrest.paging import Paged + +_LOGGER = logging.getLogger(__name__) + + +copyright_header = b"""# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# +# Code generated by Microsoft (R) AutoRest Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is +# regenerated. +# -------------------------------------------------------------------------- + +""" + +header = copyright_header+b"""from msrest.serialization import Model +from msrest.exceptions import HttpOperationError +""" + +paging_header = copyright_header+b"""from msrest.paging import Paged +""" + +init_file = """ +try: + from .{} import * +except (SyntaxError, ImportError): + from .{} import * +from .{} import * +""" + +MODEL_PY2_NAME = "_models" +MODEL_PY3_NAME = "_models_py3" +PAGED_NAME = "_paged_models" + +def as_file_name(name): + return name + ".py" + +def parse_input(input_parameter): + """From a syntax like package_name#submodule, build a package name + and complete module name. + """ + split_package_name = input_parameter.split('#') + package_name = split_package_name[0] + module_name = package_name.replace("-", ".") + if len(split_package_name) >= 2: + module_name = ".".join([module_name, split_package_name[1]]) + return package_name, module_name + +def solve_mro(models): + for models_module in models: + models_path = models_module.__path__[0] + _LOGGER.info(f"Working on {models_path}") + if Path(models_path, as_file_name(MODEL_PY3_NAME)).exists(): + _LOGGER.info("Skipping since already patched") + return + + # Build the new files in a temp folder + with tempfile.TemporaryDirectory() as temp_folder: + final_models_path = Path(temp_folder, "models") + final_models_path.mkdir() + solve_one_model(models_module, final_models_path) + + # Swith the files + shutil.rmtree(models_path) + shutil.move(final_models_path, models_path) + +def solve_one_model(models_module, output_folder): + """Will build the compacted models in the output_folder""" + + models_classes = [ + (len(model_class.__mro__), model_name, inspect.getfile(model_class), model_class) for model_name, model_class in vars(models_module).items() + if model_name[0].isupper() and Model in model_class.__mro__ + ] + # Sort on MRO size first, and then alphabetically + models_classes.sort(key=lambda x: (x[0], x[1])) + + # Just need the name of exceptions + exceptions_classes = [ + model_name for model_name, model_class in vars(models_module).items() + if model_name[0].isupper() and HttpOperationError in model_class.__mro__ + ] + + py2_models_classes = [ + (len_mro, model_name, path.replace("_py3.py", ".py"), None) + for len_mro, model_name, path, _ in models_classes + ] + + paged_models_classes = [ + (model_name, inspect.getfile(model_class), model_class) for model_name, model_class in vars(models_module).items() + if model_name[0].isupper() and Paged in model_class.__mro__ + ] + + enum_models_classes = [ + (model_name, inspect.getfile(model_class), model_class) for model_name, model_class in vars(models_module).items() + if model_name[0].isupper() and Enum in model_class.__mro__ + ] + if enum_models_classes: + # Can't be more than one enum file + enum_file = Path(enum_models_classes[0][1]) + enum_file_module_name = "_"+enum_file.with_suffix('').name + shutil.copyfile(enum_file, Path(output_folder, as_file_name(enum_file_module_name))) + else: + enum_file_module_name = None + + write_model_file(Path(output_folder, as_file_name(MODEL_PY3_NAME)), models_classes) + write_model_file(Path(output_folder, as_file_name(MODEL_PY2_NAME)), py2_models_classes) + write_paging_file(Path(output_folder, as_file_name(PAGED_NAME)), paged_models_classes) + write_complete_init( + Path(output_folder, "__init__.py"), + models_classes, + exceptions_classes, + paged_models_classes, + enum_models_classes, + enum_file_module_name + ) + +def write_model_file(output_file_path, classes_to_write): + with open(output_file_path, "bw") as write_fd: + write_fd.write(header) + + for model in classes_to_write: + _, _, model_file_path, _ = model + + with open(model_file_path, "rb") as read_fd: + lines = read_fd.readlines() + # Skip until it's "class XXXX" + while lines: + if lines[0].startswith(b"class "): + break + lines.pop(0) + else: + raise ValueError("Never found any class definition!") + # Now I keep everything + write_fd.write(b'\n') + write_fd.write(b'\n') + write_fd.writelines(lines) + +def write_paging_file(output_file_path, classes_to_write): + with open(output_file_path, "bw") as write_fd: + write_fd.write(paging_header) + + for model in classes_to_write: + _, model_file_path, _ = model + + with open(model_file_path, "rb") as read_fd: + # Skip the first 15 lines (based on Autorest deterministic behavior) + # If we want this less random, look for the first line starts with "class" + lines = read_fd.readlines()[14:] + write_fd.write(b'\n') + write_fd.write(b'\n') + write_fd.writelines(lines) + +def write_init(output_file_path, model_file_name, model_file_name_py2, paging_file_name, enum_file_name): + with open(output_file_path, "bw") as write_fd: + write_fd.write(copyright_header) + + write_fd.write(init_file.format( + model_file_name, + model_file_name_py2, + paging_file_name, + ).encode('utf8')) + if enum_file_name: + write_fd.write( + "from .{} import *".format(enum_file_name).encode('utf8') + ) + +def write_complete_init(output_file_path, models, exceptions_classes, paging_models, enum_models, enum_file_module_name): + with open(output_file_path, "bw") as write_fd: + write_fd.write(copyright_header) + + write_fd.write(b"\ntry:\n") + # Write py3 import + for _, model_name, _, _ in models: + write_fd.write(f" from .{MODEL_PY3_NAME} import {model_name}\n".encode("utf-8")) + # Py 3 exceptions + for model_name in exceptions_classes: + write_fd.write(f" from .{MODEL_PY3_NAME} import {model_name}\n".encode("utf-8")) + + write_fd.write(b"except (SyntaxError, ImportError):\n") + + # Write py2 import + for _, model_name, _, _ in models: + write_fd.write(f" from .{MODEL_PY2_NAME} import {model_name}\n".encode("utf-8")) + # Py 2 exceptions + for model_name in exceptions_classes: + write_fd.write(f" from .{MODEL_PY2_NAME} import {model_name}\n".encode("utf-8")) + + # Write paged import + for model_name, _, _ in paging_models: + write_fd.write(f"from .{PAGED_NAME} import {model_name}\n".encode("utf-8")) + + if enum_models: + # Write enum model import + for model_name, _, _ in enum_models: + write_fd.write(f"from .{enum_file_module_name} import {model_name}\n".encode("utf-8")) + + write_fd.write(b"\n\n__all__=[\n") + # Write all classes name + for _, model_name, _, _ in models: + write_fd.write(f" '{model_name}',\n".encode("utf-8")) + for model_name in exceptions_classes: + write_fd.write(f" '{model_name}',\n".encode("utf-8")) + for model_name, _, _ in paging_models: + write_fd.write(f" '{model_name}',\n".encode("utf-8")) + if enum_models: + for model_name, _, _ in enum_models: + write_fd.write(f" '{model_name}',\n".encode("utf-8")) + + write_fd.write(b"]\n") + + +def find_models_to_change(module_name): + """Will figure out if the package is a multi-api one, + and understand what to generate. + """ + main_module = importlib.import_module(module_name) + try: + models_module = main_module.models + models_module.__path__ + # It didn't fail, that's a single API package + return [models_module] + except AttributeError: + # This means I loaded the fake module "models" + # and it's multi-api, load all models + return [ + importlib.import_module('.'+label+'.models', main_module.__name__) + for (_, label, ispkg) in pkgutil.iter_modules(main_module.__path__) + if ispkg + ] + + +def find_autorest_generated_folder(module_prefix="azure.mgmt"): + """Find all Autorest generated code in that module prefix. + + This actually looks for a "models" package only. We could be smarter if necessary. + """ + _LOGGER.info(f"Looking for Autorest generated package in {module_prefix}") + result = [] + try: + _LOGGER.debug(f"Try {module_prefix}") + importlib.import_module(".models", module_prefix) + # If not exception, we found it + _LOGGER.info(f"Found {module_prefix}") + result.append(module_prefix) + except ModuleNotFoundError: + # No model, might dig deeper + prefix_module = importlib.import_module(module_prefix) + for _, sub_package, ispkg in pkgutil.iter_modules(prefix_module.__path__, module_prefix+"."): + if ispkg: + result += find_autorest_generated_folder(sub_package) + return result + + +def main(prefix="azure.mgmt"): + packages = find_autorest_generated_folder(prefix) + for autorest_package in packages: + models_module = find_models_to_change(autorest_package) + solve_mro(models_module) + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main(sys.argv[1] if len(sys.argv) >= 2 else "azure.mgmt") \ No newline at end of file