regenerate sdk pipeline (#2686)

* sdk regeneration pipeline

* fix

* fix typespec folder

* fix

* add trigger

* fix yaml

* fix yaml

* fix commit

* fix yaml

* fix yaml to install pnpm

* use higher node version

* update steps

* update steps

* update yaml

* update yaml

* update yaml

* update yaml

* update yaml

* update yaml

* update yaml

* record timing

* update yaml

* update yaml

* for all sdk

* update fail_to_regenearte

eng/pipelines/sdk_regenerate.yaml

@@ -0,0 +1,79 @@
+trigger:
+  branches:
+    include:
+      - main
+
+schedules:
+- cron: '0 0 * * *'
+  displayName: Daily midnight build to regenerate SDK
+  branches:
+    include:
+    - main
+
+pr: none
+
+pool:
+  name: "azsdk-pool-mms-ubuntu-2004-general"
+  vmImage: "MMSUbuntu20.04"
+
+variables:
+  - group: Release Secrets for GitHub
+
+
+resources:
+  repositories:
+    - repository: azure-sdk-for-python
+      type: github
+      name: Azure/azure-sdk-for-python
+      endpoint: azure
+      ref: main
+
+jobs:
+- job: Generate_SDK
+
+  timeoutInMinutes: 120
+
+  steps:
+  - checkout: self
+    fetchDepth: 1
+  - checkout: azure-sdk-for-python
+
+  - task: NodeTool@0
+    displayName: 'Install Node.js 18.x'
+    inputs:
+      versionSpec: '18.x'
+
+  - script: npm install -g [email protected]
+    displayName: Install pnpm 9.5.0
+
+  - script: pnpm install
+    displayName: pnpm install
+    workingDirectory: $(Build.SourcesDirectory)/autorest.python
+
+  - script: pnpm build
+    displayName: pnpm build
+    workingDirectory: $(Build.SourcesDirectory)/autorest.python
+
+  - script: npm pack
+    displayName: npm pack
+    workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/typespec-python
+
+  - script: npm install -g @azure-tools/typespec-client-generator-cli
+    displayName: 'Install tsp-client'
+
+  - script: |
+      python3 $(Build.SourcesDirectory)/autorest.python/eng/scripts/sdk_regenerate.py --sdk-root=$(Build.SourcesDirectory)/azure-sdk-for-python --typespec-python-root=$(Build.SourcesDirectory)/autorest.python --typespec-python-branch=$(Build.SourceBranchName)
+    displayName: 'Generate SDK'
+    workingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-python
+
+  - template: /eng/common/pipelines/templates/steps//create-pull-request.yml@azure-sdk-for-python
+    parameters:
+      WorkingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-python
+      ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-for-python/eng/common/scripts
+      RepoName: azure-sdk-for-python
+      BaseBranchName: 'refs/heads/main'
+      PRBranchName: typespec-python-$(Build.SourceBranchName)
+      CommitMsg: 'Regenerate SDK based on typespec-python branch $(Build.SourceBranchName)'
+      PRTitle: '[Automation] Regenerate SDK based on typespec-python branch $(Build.SourceBranchName)'
+      OpenAsDraft: 'true'
+      PushArgs: '--force'

eng/scripts/sdk_regenerate.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+from typing import Dict, List
+from pathlib import Path
+import subprocess
+from datetime import datetime
+from subprocess import check_call, check_output
+import argparse
+import logging
+import json
+
+
+def update_emitter_package(sdk_root: str, typespec_python_root: str):
+    # find the typespec-python.tgz
+    typespec_python_tgz = None
+    for item in (Path(typespec_python_root) / "packages/typespec-python").iterdir():
+        if "typespec-python" in item.name and item.name.endswith(".tgz"):
+            typespec_python_tgz = item
+            break
+    if not typespec_python_tgz:
+        logging.error("can not find .tgz for typespec-python")
+        raise FileNotFoundError("can not find .tgz for typespec-python")
+
+    # update the emitter-package.json
+    emitter_package_folder = Path(sdk_root) / "eng/emitter-package.json"
+    with open(emitter_package_folder, "r") as f:
+        emitter_package = json.load(f)
+    emitter_package["dependencies"]["@azure-tools/typespec-python"] = typespec_python_tgz.absolute().as_posix()
+    with open(emitter_package_folder, "w") as f:
+        json.dump(emitter_package, f, indent=2)
+
+    # update the emitter-package-lock.json
+    try:
+        check_call("tsp-client --generate-lock-file", shell=True)
+    except Exception as e:
+        logging.error("failed to update emitter-package-lock.json")
+        logging.error(e)
+        raise e
+
+
+def regenerate_sdk() -> Dict[str, List[str]]:
+    result = {"succeed_to_regenerate": [], "fail_to_regenerate": [], "time_to_regenerate": str(datetime.now())}
+    # get all tsp-location.yaml
+    for item in Path(".").rglob("tsp-location.yaml"):
+        package_folder = item.parent
+        try:
+            output = (
+                check_output("tsp-client update", shell=True, cwd=str(package_folder), stderr=subprocess.STDOUT)
+                .decode("utf-8")
+                .split("\n")
+            )
+            errors = [line for line in output if "- error " in line.lower()]
+            if errors:
+                raise Exception("\n".join(errors))
+        except Exception as e:
+            logging.error(f"failed to regenerate {package_folder.name}")
+            logging.error(e)
+            result["fail_to_regenerate"].append(package_folder.name)
+        else:
+            result["succeed_to_regenerate"].append(package_folder.name)
+    return result
+
+
+def checkout_branch(branch: str):
+    try:
+        check_call(f"git fetch azure-sdk {branch}", shell=True)
+        check_call(f"git checkout {branch}", shell=True)
+    except Exception:
+        check_call(f"git checkout -b {branch}", shell=True)
+
+
+def prepare_branch(typespec_python_branch: str):
+    check_call("git remote add azure-sdk https://github.com/azure-sdk/azure-sdk-for-python.git", shell=True)
+    checkout_branch("typespec-python-main")
+
+    if typespec_python_branch != "main":
+        checkout_branch(f"typespec-python-{typespec_python_branch}")
+
+
+def git_add():
+    check_call("git add .", shell=True)
+
+
+def main(sdk_root: str, typespec_python_root: str, typespec_python_branch: str):
+    prepare_branch(typespec_python_branch)
+    update_emitter_package(sdk_root, typespec_python_root)
+    result = regenerate_sdk()
+    with open("aaaa-regenerate-sdk-result.json", "w") as f:
+        json.dump(result, f, indent=2)
+    git_add()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="SDK regeneration")
+
+    parser.add_argument(
+        "--sdk-root",
+        help="SDK repo root folder",
+        type=str,
+    )
+
+    parser.add_argument(
+        "--typespec-python-root",
+        help="typespec-python repo root folder",
+        type=str,
+    )
+
+    parser.add_argument(
+        "--typespec-python-branch",
+        help="branch of typespec-python repo",
+        type=str,
+    )
+
+    args = parser.parse_args()
+
+    main(args.sdk_root, args.typespec_python_root, args.typespec_python_branch)