diff --git a/eng/pipelines/sdk_regenerate.yaml b/eng/pipelines/sdk_regenerate.yaml new file mode 100644 index 0000000000..77b726f697 --- /dev/null +++ b/eng/pipelines/sdk_regenerate.yaml @@ -0,0 +1,79 @@ +trigger: + branches: + include: + - main + +schedules: +- cron: '0 0 * * *' + displayName: Daily midnight build to regenerate SDK + branches: + include: + - main + +pr: none + +pool: + name: "azsdk-pool-mms-ubuntu-2004-general" + vmImage: "MMSUbuntu20.04" + +variables: + - group: Release Secrets for GitHub + + +resources: + repositories: + - repository: azure-sdk-for-python + type: github + name: Azure/azure-sdk-for-python + endpoint: azure + ref: main + +jobs: +- job: Generate_SDK + + timeoutInMinutes: 120 + + steps: + - checkout: self + fetchDepth: 1 + - checkout: azure-sdk-for-python + + - task: NodeTool@0 + displayName: 'Install Node.js 18.x' + inputs: + versionSpec: '18.x' + + - script: npm install -g pnpm@9.5.0 + displayName: Install pnpm 9.5.0 + + - script: pnpm install + displayName: pnpm install + workingDirectory: $(Build.SourcesDirectory)/autorest.python + + - script: pnpm build + displayName: pnpm build + workingDirectory: $(Build.SourcesDirectory)/autorest.python + + - script: npm pack + displayName: npm pack + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/typespec-python + + - script: npm install -g @azure-tools/typespec-client-generator-cli + displayName: 'Install tsp-client' + + - script: | + python3 $(Build.SourcesDirectory)/autorest.python/eng/scripts/sdk_regenerate.py --sdk-root=$(Build.SourcesDirectory)/azure-sdk-for-python --typespec-python-root=$(Build.SourcesDirectory)/autorest.python --typespec-python-branch=$(Build.SourceBranchName) + displayName: 'Generate SDK' + workingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-python + + - template: /eng/common/pipelines/templates/steps//create-pull-request.yml@azure-sdk-for-python + parameters: + WorkingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-python + ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-for-python/eng/common/scripts + RepoName: azure-sdk-for-python + BaseBranchName: 'refs/heads/main' + PRBranchName: typespec-python-$(Build.SourceBranchName) + CommitMsg: 'Regenerate SDK based on typespec-python branch $(Build.SourceBranchName)' + PRTitle: '[Automation] Regenerate SDK based on typespec-python branch $(Build.SourceBranchName)' + OpenAsDraft: 'true' + PushArgs: '--force' diff --git a/eng/scripts/sdk_regenerate.py b/eng/scripts/sdk_regenerate.py new file mode 100644 index 0000000000..acd6418618 --- /dev/null +++ b/eng/scripts/sdk_regenerate.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from typing import Dict, List +from pathlib import Path +import subprocess +from datetime import datetime +from subprocess import check_call, check_output +import argparse +import logging +import json + + +def update_emitter_package(sdk_root: str, typespec_python_root: str): + # find the typespec-python.tgz + typespec_python_tgz = None + for item in (Path(typespec_python_root) / "packages/typespec-python").iterdir(): + if "typespec-python" in item.name and item.name.endswith(".tgz"): + typespec_python_tgz = item + break + if not typespec_python_tgz: + logging.error("can not find .tgz for typespec-python") + raise FileNotFoundError("can not find .tgz for typespec-python") + + # update the emitter-package.json + emitter_package_folder = Path(sdk_root) / "eng/emitter-package.json" + with open(emitter_package_folder, "r") as f: + emitter_package = json.load(f) + emitter_package["dependencies"]["@azure-tools/typespec-python"] = typespec_python_tgz.absolute().as_posix() + with open(emitter_package_folder, "w") as f: + json.dump(emitter_package, f, indent=2) + + # update the emitter-package-lock.json + try: + check_call("tsp-client --generate-lock-file", shell=True) + except Exception as e: + logging.error("failed to update emitter-package-lock.json") + logging.error(e) + raise e + + +def regenerate_sdk() -> Dict[str, List[str]]: + result = {"succeed_to_regenerate": [], "fail_to_regenerate": [], "time_to_regenerate": str(datetime.now())} + # get all tsp-location.yaml + for item in Path(".").rglob("tsp-location.yaml"): + package_folder = item.parent + try: + output = ( + check_output("tsp-client update", shell=True, cwd=str(package_folder), stderr=subprocess.STDOUT) + .decode("utf-8") + .split("\n") + ) + errors = [line for line in output if "- error " in line.lower()] + if errors: + raise Exception("\n".join(errors)) + except Exception as e: + logging.error(f"failed to regenerate {package_folder.name}") + logging.error(e) + result["fail_to_regenerate"].append(package_folder.name) + else: + result["succeed_to_regenerate"].append(package_folder.name) + return result + + +def checkout_branch(branch: str): + try: + check_call(f"git fetch azure-sdk {branch}", shell=True) + check_call(f"git checkout {branch}", shell=True) + except Exception: + check_call(f"git checkout -b {branch}", shell=True) + + +def prepare_branch(typespec_python_branch: str): + check_call("git remote add azure-sdk https://github.com/azure-sdk/azure-sdk-for-python.git", shell=True) + checkout_branch("typespec-python-main") + + if typespec_python_branch != "main": + checkout_branch(f"typespec-python-{typespec_python_branch}") + + +def git_add(): + check_call("git add .", shell=True) + + +def main(sdk_root: str, typespec_python_root: str, typespec_python_branch: str): + prepare_branch(typespec_python_branch) + update_emitter_package(sdk_root, typespec_python_root) + result = regenerate_sdk() + with open("aaaa-regenerate-sdk-result.json", "w") as f: + json.dump(result, f, indent=2) + git_add() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="SDK regeneration") + + parser.add_argument( + "--sdk-root", + help="SDK repo root folder", + type=str, + ) + + parser.add_argument( + "--typespec-python-root", + help="typespec-python repo root folder", + type=str, + ) + + parser.add_argument( + "--typespec-python-branch", + help="branch of typespec-python repo", + type=str, + ) + + args = parser.parse_args() + + main(args.sdk_root, args.typespec_python_root, args.typespec_python_branch)