diff --git a/packaged_releases/README.md b/packaged_releases/README.md index e6ff26e2bd5..81d6e456658 100644 --- a/packaged_releases/README.md +++ b/packaged_releases/README.md @@ -3,67 +3,45 @@ Creating a Packaged Release This document provides instructions on creating a packaged release. -Throughout this document, `{VERSION}` refers to a semantic version for this packaged release (e.g. `0.1.5`). +Throughout this document, `{VERSION}` refers to a semantic version for this packaged release (e.g. `0.2.3`). -1 - Assemble module source code -------------------------------- - -To assemble the source code, visit [Releases](https://github.com/Azure/azure-cli/releases) and download the source code for each component (and version) that should be part of the packaged release. +1 - Create release archive +-------------------------- -Use the downloaded source code to create a directory with the structure outlined below. +We create a `.tar.gz` containing the source code for the packaged release. +This archive will be used as the basis for the Docker, Debian and Homebrew builds as they build from this source. -Make a directory with name `azure-cli_packaged_{VERSION}`: +Clone the repo afresh: ``` -$ mkdir azure-cli_packaged_{VERSION} +$ git clone https://github.com/azure/azure-cli ``` -**IMPORTANT** - The builds expect a certain folder structure. -Expected folder structure inside of `azure-cli_packaged_{VERSION}`: +Run the script to create the release archive from the 'scripts' folder at the repo root: ``` -. -|-- az.completion -|-- src -| |-- azure-cli -| |-- setup.py -| `-- etc... -| |-- azure-cli-core -| |-- setup.py -| `-- etc... -| |-- azure-cli-nspkg -| |-- setup.py -| `-- etc... -| |-- command_modules -| |-- -| |-- setup.py -| `-- etc... +$ cd scripts +$ python -m automation.release.packaged --version {VERSION} --components azure-cli=VERSION ... ``` -(A script may be available in the future to make this step more straightforward.) - -Notes: -- Only the packages that will be in the CLI should be included here; leave out 'optional' components unless there's a specific reason to include any extra components. -- Make sure the versions of components don't include the `+dev` suffix. Remove these if this is the case. - -APPLY ANY PATCHES: -Modify the file in question in the directory created from (You can use the `patch_*` files in `patches` subdirectory for this). - - -2 - Create release archive --------------------------- - -We create a `.tar.gz` containing the source code for the packaged release. -This archive will be used as the basis for the Docker, Debian and Homebrew builds as they build from this source. +A full example: +``` +$ cat ~/cli-components.json +{ + "azure-cli": "2.0.1", + "azure-cli-core": "2.0.1", + "azure-cli-component": "2.0.0", + "azure-cli-acs": "2.0.0" +} +$ python -m automation.release.packaged --version 0.2.3 -f ~/cli-components.json +``` -The archive should have the following name `azure-cli_packaged_{VERSION}.tar.gz`. +OR -Archive the assembled source code from above: ``` -$ tar -cvzf azure-cli_packaged_{VERSION}.tar.gz azure-cli_packaged_{VERSION}/ +$ python -m automation.release.packaged --version 0.2.3 --components azure-cli=2.0.1 acs=2.0.1 appservice=0.1.1b6 batch=0.1.1b5 cloud=2.0.0 component=2.0.0 configure=2.0.1 container=0.1.1b4 core=2.0.1 documentdb=0.1.1b2 feedback=2.0.0 find=0.0.1b1 iot=0.1.1b3 keyvault=0.1.1b6 network=2.0.1 nspkg=2.0.0 profile=2.0.1 redis=0.1.1b3 resource=2.0.1 role=2.0.0 sql=0.1.1b6 storage=2.0.1 vm=2.0.1 ``` - -3 - Upload release archive +2 - Upload release archive -------------------------- The release archive should be uploaded to a storage account. @@ -86,13 +64,13 @@ Get the SHA256 checksum: $ shasum -a 256 azure-cli_packaged_{VERSION}.tar.gz ``` -4 - Build/Release for Debian, Docker, Homebrew +3 - Build/Release for Debian, Docker, Homebrew ---------------------------------------------- Follow the instructions in the `debian`, `docker` and `homebrew` subdirectories to create these releases. -5 - Modify HISTORY.md +4 - Modify HISTORY.md --------------------- Modify the packaged release history with release notes on this release and create a PR for this change. diff --git a/scripts/automation/release/packaged.py b/scripts/automation/release/packaged.py new file mode 100644 index 00000000000..efd26883ef8 --- /dev/null +++ b/scripts/automation/release/packaged.py @@ -0,0 +1,137 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from __future__ import print_function + +import argparse +import os +import sys +import tempfile +import tarfile +import shutil +import json +from subprocess import check_call, check_output + +from .version_patcher import VersionPatcher +from ..utilities.path import get_all_module_paths, get_repo_root + +REPO_ROOT_DIR = get_repo_root() +COMPLETION_FILE = os.path.join(REPO_ROOT_DIR, 'packaged_releases', 'az.completion') +ARCHIVE_FILE_TMPL = 'azure-cli_packaged_{}' + + +class Patch(object): # pylint: disable=too-few-public-methods + def __init__(self, src_of_patch, path_to_patch): + """ + - src: Relative path from the repo root + - dest: Relative path to file to patch in the packaged release + """ + self.src_of_patch = src_of_patch + self.path_to_patch = path_to_patch + + def apply(self, working_dir): + src = os.path.join(REPO_ROOT_DIR, self.src_of_patch) + dest = os.path.join(working_dir, self.path_to_patch) + shutil.copy(src, dest) + + +PATCHES = [ + Patch(os.path.join('packaged_releases', 'patches', 'patch_pkg_util.py'), + os.path.join('src', 'azure-cli-core', 'azure', 'cli', 'core', '_pkg_util.py')), + Patch(os.path.join('packaged_releases', 'patches', 'patch_component_custom.py'), + os.path.join('src', 'command_modules', 'azure-cli-component', 'azure', 'cli', + 'command_modules', 'component', 'custom.py')) + ] + + +def error_exit(msg): + print('ERROR: '+msg, file=sys.stderr) + sys.exit(1) + + +def _gen_tag(c_name, c_version): + return '{}-{}'.format(c_name, c_version) + + +def _verified_tags(components): + available_tags = check_output(['git', 'tag'], cwd=REPO_ROOT_DIR).split() + for c_name, c_version in components: + t = _gen_tag(c_name, c_version) + if t not in available_tags: + print('Tag {} not found.'.format(t)) + return False + return True + + +def create_packaged_archive(version, components, archive_dest=None, use_version_patch=True): + # Verify the components and versions by checking git tags + if not _verified_tags(components): + error_exit('Some components or versions are not valid.') + working_dir = tempfile.mkdtemp() + print('Using tmp directory {}'.format(working_dir)) + modules = {n: p for n, p in get_all_module_paths()} + cur_git_commitish = check_output(['git', 'rev-parse', 'HEAD'], cwd=REPO_ROOT_DIR).strip() + for c_name, c_version in components: + c_path = modules[c_name] + git_tag = _gen_tag(c_name, c_version) + check_call(['git', 'checkout', git_tag], cwd=REPO_ROOT_DIR) + patcher = VersionPatcher(use_version_patch, c_name, c_path) + patcher.patch() + sub_dir = 'command_modules' if 'command_modules' in c_path else '' + shutil.copytree(c_path, os.path.join(working_dir, 'src', sub_dir, c_name)) + patcher.unpatch() + check_call(['git', 'checkout', cur_git_commitish], cwd=REPO_ROOT_DIR) + # Add completion file + completion_dest = os.path.join(working_dir, 'az.completion') + shutil.copy(COMPLETION_FILE, completion_dest) + # Apply patches + for patch in PATCHES: + patch.apply(working_dir) + # Build archive + archive_filename = ARCHIVE_FILE_TMPL.format(version) + archive_dest = os.path.expanduser(archive_dest) or os.getcwd() + archive_path = os.path.join(archive_dest, archive_filename+'.tar.gz') + with tarfile.open(archive_path, 'w:gz') as tar: + tar.add(working_dir, arcname=archive_filename) + print("Archive saved to {}".format(archive_path)) + print("Done.") + + +def _type_components_list(value): + c_name, c_version = value.split('=', 1) + if not c_name.startswith('azure-cli'): + c_name = 'azure-cli-' + c_name + return (c_name, c_version) + + +def _type_json_file(value): + with open(os.path.expanduser(value)) as open_file: + data = json.load(open_file) + return [(k, data[k]) for k in data] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="Automated generation of the packaged release archive.") + + group = parser.add_mutually_exclusive_group(required=True) + + group.add_argument('--components', '-c', nargs='+', + help="Space separated list in 'component=version' format. " + "(e.g. azure-cli=2.0.0 vm=2.0.0)", + type=_type_components_list) + group.add_argument('--file-data', '-f', + help='Path to JSON file with commands in key/value format. ' + '(e.g. {"azure-cli":"2.0.0", ...})', + type=_type_json_file) + parser.add_argument('--version', '-v', required=True, + help="The version to name the packaged release.") + parser.add_argument('--dest', '-d', + help="The destination directory to place the archive. " + "Defaults to current directory.") + args = parser.parse_args() + + components_list = args.components or args.file_data + create_packaged_archive(args.version, components_list, args.dest)