Skip to content

Commit b6d65fa

Browse files
msyycmsyyc
authored
[T2] python track2 new pipeline (#16374)
* track2 new pipeline * Update swagger_to_sdk_config_autorest.json * Update swagger_to_sdk_config_autorest.json * fix comment * better display ui
1 parent db32606 commit b6d65fa

File tree

12 files changed

+370
-30
lines changed

12 files changed

+370
-30
lines changed

‎ci_template.yml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# DO NOT EDIT THIS FILE
2+
# This file is generated automatically and any changes will be lost.
3+
4+
trigger:
5+
branches:
6+
include:
7+
- master
8+
- hotfix/*
9+
- release/*
10+
- restapi*
11+
paths:
12+
include:
13+
- sdk/MyService/
14+
15+
pr:
16+
branches:
17+
include:
18+
- master
19+
- feature/*
20+
- hotfix/*
21+
- release/*
22+
- restapi*
23+
paths:
24+
include:
25+
- sdk/MyService/
26+
27+
extends:
28+
template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
29+
parameters:
30+
ServiceDirectory: MyService
31+
Artifacts:
32+
- name: azure_mgmt_MyService
33+
safeName: azuremgmtMyService

‎scripts/automation_generate.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
VIRTUAL_ENV=$TMPDIR/venv-sdk
4+
export VIRTUAL_ENV
5+
PATH="$VIRTUAL_ENV/bin:$PATH"
6+
export PATH
7+
8+
# node version degrade
9+
sudo npm install -g n
10+
sudo n 10.15.0
11+
export PATH=/usr/local/n/versions/node/10.15.0/bin:$PATH
12+
13+
# generate code and package
14+
python -m packaging_tools.auto_codegen "$1" "$TMPDIR/venv-sdk/auto_temp.json" 2>&1
15+
echo "[Generate] codegen done!!!"
16+
python -m packaging_tools.auto_package "$TMPDIR/venv-sdk/auto_temp.json" "$2" 2>&1
17+
echo "[Generate] generate done!!!"
18+
19+
if [ ! -f "$2" ]; then
20+
echo "$2 does not exist!!!"
21+
exit 1
22+
fi

‎scripts/automation_init.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
rm -rf $TMPDIR/venv-sdk
4+
python3 -m venv $TMPDIR/venv-sdk
5+
VIRTUAL_ENV=$TMPDIR/venv-sdk
6+
export VIRTUAL_ENV
7+
PATH="$VIRTUAL_ENV/bin:$PATH"
8+
export PATH
9+
python scripts/dev_setup.py -p azure-core
10+
echo "{}" >> $2
11+
echo "[Generate] init success!!!"

‎swagger_to_sdk_config.json‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,26 @@
11
{
2-
"$schema": "https://openapistorageprod.blob.core.windows.net/sdkautomation/prod/schemas/swagger_to_sdk_config.schema.json",
3-
"meta": {
4-
"autorest_options": {
5-
"version": "3.0.6272",
6-
"use": "@autorest/[email protected]",
7-
"python": "",
8-
"python-mode": "update",
9-
"sdkrel:python-sdks-folder": "./sdk/.",
10-
"multiapi": "",
11-
"track2": ""
2+
"advancedOptions": {
3+
"createSdkPullRequests": true,
4+
"generationCallMode": "one-for-all-configs"
5+
},
6+
7+
"initOptions": {
8+
"initScript": {
9+
"path": "sh scripts/automation_init.sh"
10+
}
11+
},
12+
13+
"generateOptions": {
14+
"generateScript": {
15+
"path": "sh scripts/automation_generate.sh",
16+
"stderr": {
17+
"showInComment": true
18+
},
19+
"stdout": {
20+
"showInComment": "^\\[Autorest\\]"
21+
}
1222
},
13-
"advanced_options": {
14-
"create_sdk_pull_requests": true,
15-
"sdk_generation_pull_request_base": "integration_branch"
16-
},
17-
"repotag": "azure-sdk-for-python-track2",
18-
"version": "0.2.0"
23+
24+
"parseGenerateOutput": true
1925
}
20-
}
26+
}

‎swagger_to_sdk_config_autorest.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"meta": {
3+
"autorest_options": {
4+
"version": "3.0.6272",
5+
"use": "@autorest/[email protected]",
6+
"python": "",
7+
"python-mode": "update",
8+
"sdkrel:python-sdks-folder": "./sdk/.",
9+
"multiapi": "",
10+
"track2": ""
11+
},
12+
"advanced_options": {
13+
"create_sdk_pull_requests": true,
14+
"sdk_generation_pull_request_base": "integration_branch"
15+
},
16+
"repotag": "azure-sdk-for-python-track2",
17+
"version": "0.2.0"
18+
}
19+
}

‎tools/azure-devtools/src/azure_devtools/ci_tools/git_tools.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,11 @@ def get_diff_file_list(git_folder):
9595
repo = Repo(str(git_folder))
9696
output = repo.git.diff("--name-only")
9797
return output.splitlines()
98+
99+
def get_add_diff_file_list(git_folder):
100+
"""List of new files.
101+
"""
102+
repo = Repo(str(git_folder))
103+
repo.git.add("sdk")
104+
output = repo.git.diff("HEAD", "--name-only")
105+
return output.splitlines()

‎tools/azure-sdk-tools/packaging_tools/auto_codegen.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import argparse
2+
import json
3+
import logging
4+
from pathlib import Path
5+
import re
6+
from subprocess import check_call
7+
8+
from .swaggertosdk.SwaggerToSdkCore import (
9+
CONFIG_FILE,
10+
)
11+
from azure_devtools.ci_tools.git_tools import get_add_diff_file_list
12+
from .generate_sdk import generate
13+
14+
_LOGGER = logging.getLogger(__name__)
15+
_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII)
16+
17+
DEFAULT_DEST_FOLDER = "./dist"
18+
19+
20+
def get_package_names(sdk_folder):
21+
files = get_add_diff_file_list(sdk_folder)
22+
matches = {_SDK_FOLDER_RE.search(f) for f in files}
23+
package_names = {match.groups() for match in matches if match is not None}
24+
return package_names
25+
26+
27+
def init_new_service(package_name, folder_name):
28+
ci = Path(folder_name, 'ci.yml')
29+
if not ci.exists():
30+
check_call(f'python -m packaging_tools --build-conf {package_name} -o {folder_name}', shell=True)
31+
with open('ci_template.yml', 'r') as file_in:
32+
content = file_in.readlines()
33+
name = package_name.replace('azure-', '').replace('mgmt-', '')
34+
content = [line.replace('MyService', name) for line in content]
35+
with open(str(ci), 'w') as file_out:
36+
file_out.writelines(content)
37+
38+
39+
def main(generate_input, generate_output):
40+
with open(generate_input, "r") as reader:
41+
data = json.load(reader)
42+
43+
spec_folder = data['specFolder']
44+
sdk_folder = "."
45+
result = {}
46+
package_total = set()
47+
for input_readme in data["relatedReadmeMdFiles"]:
48+
relative_path_readme = str(Path(spec_folder, input_readme))
49+
_LOGGER.info(f'[CODEGEN]({input_readme})codegen begin')
50+
generate(CONFIG_FILE,
51+
sdk_folder,
52+
[],
53+
relative_path_readme,
54+
spec_folder,
55+
force_generation=True
56+
)
57+
package_names = get_package_names(sdk_folder)
58+
_LOGGER.info(f'[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]')
59+
60+
for folder_name, package_name in package_names:
61+
if package_name in package_total:
62+
continue
63+
64+
package_total.add(package_name)
65+
if package_name not in result:
66+
package_entry = {}
67+
package_entry['packageName'] = package_name
68+
package_entry["path"] = [folder_name]
69+
package_entry['readmeMd'] = [input_readme]
70+
result[package_name] = package_entry
71+
else:
72+
result[package_name]["path"].append(folder_name)
73+
result[package_name]["readmeMd"].append(input_readme)
74+
75+
# Generate some necessary file for new service
76+
init_new_service(package_name, folder_name)
77+
78+
# Setup package locally
79+
check_call(f'pip install --ignore-requires-python -e {str(Path(sdk_folder, folder_name, package_name))}',
80+
shell=True)
81+
82+
83+
# remove duplicates
84+
for value in result.values():
85+
value['path'] = list(set(value['path']))
86+
value['readmeMd'] = list(set(value['readmeMd']))
87+
88+
with open(generate_output, "w") as writer:
89+
json.dump(result, writer)
90+
91+
92+
def generate_main():
93+
"""Main method"""
94+
95+
parser = argparse.ArgumentParser(
96+
description='Build SDK using Autorest, offline version.',
97+
formatter_class=argparse.RawTextHelpFormatter)
98+
parser.add_argument('generate_input',
99+
help='Generate input file path')
100+
parser.add_argument('generate_output',
101+
help='Generate output file path')
102+
parser.add_argument("-v", "--verbose",
103+
dest="verbose", action="store_true",
104+
help="Verbosity in INFO mode")
105+
parser.add_argument("--debug",
106+
dest="debug", action="store_true",
107+
help="Verbosity in DEBUG mode")
108+
parser.add_argument("-c", "--codegen",
109+
dest="debug", action="store_true",
110+
help="Verbosity in DEBUG mode")
111+
112+
args = parser.parse_args()
113+
main_logger = logging.getLogger()
114+
logging.basicConfig()
115+
main_logger.setLevel(logging.DEBUG if args.verbose or args.debug else logging.INFO)
116+
117+
main(args.generate_input, args.generate_output)
118+
119+
120+
if __name__ == "__main__":
121+
generate_main()

‎tools/azure-sdk-tools/packaging_tools/auto_package.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import argparse
2+
import json
3+
import glob
4+
import logging
5+
import os
6+
from pathlib import Path
7+
import re
8+
from subprocess import check_call
9+
10+
from azure_devtools.ci_tools.git_tools import get_diff_file_list
11+
from .change_log import main as change_log_main
12+
13+
_LOGGER = logging.getLogger(__name__)
14+
_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII)
15+
16+
DEFAULT_DEST_FOLDER = "./dist"
17+
18+
19+
def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
20+
# a package will exist in either one, or the other folder. this is why we can resolve both at the same time.
21+
absdirs = [os.path.dirname(package) for package in
22+
(glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))]
23+
24+
absdirpath = os.path.abspath(absdirs[0])
25+
check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
26+
check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
27+
28+
29+
def get_package_names(sdk_folder):
30+
files = get_diff_file_list(sdk_folder)
31+
matches = {_SDK_FOLDER_RE.search(f) for f in files}
32+
package_names = {match.groups() for match in matches if match is not None}
33+
return package_names
34+
35+
36+
def change_log_generate(package_name):
37+
from pypi_tools.pypi import PyPIClient
38+
client = PyPIClient()
39+
try:
40+
client.get_ordered_versions(package_name)
41+
except:
42+
return " - Initial Release"
43+
else:
44+
return change_log_main(f"{package_name}:pypi", f"{package_name}:latest")
45+
46+
47+
def main(generate_input, generate_output):
48+
with open(generate_input, "r") as reader:
49+
data = json.load(reader)
50+
if not data:
51+
return
52+
53+
sdk_folder = '.'
54+
result = {
55+
'packages': []
56+
}
57+
for package in data.values():
58+
package_name = package['packageName']
59+
# Changelog
60+
md_output = change_log_generate(package_name)
61+
package["changelog"] = {
62+
"content": md_output,
63+
"hasBreakingChange": "Breaking changes" in md_output or "Initial Release" in md_output
64+
}
65+
_LOGGER.info(f'[PACKAGE]({package_name})[CHANGELOG]:{md_output}')
66+
# Built package
67+
create_package(package_name)
68+
folder_name = package['path'][0]
69+
dist_path = Path(sdk_folder, folder_name, package_name, "dist")
70+
package["artifacts"] = [
71+
str(dist_path / package_file) for package_file in os.listdir(dist_path)
72+
]
73+
# Installation package
74+
package["installInstructions"] = {
75+
"full": "You can install the use using pip install of the artificats.",
76+
"lite": f"pip install {package_name}"
77+
}
78+
package["result"]: "success"
79+
result['packages'].append(package)
80+
81+
with open(generate_output, "w") as writer:
82+
json.dump(result, writer)
83+
84+
85+
def generate_main():
86+
"""Main method"""
87+
88+
parser = argparse.ArgumentParser(
89+
description='Build SDK using Autorest, offline version.',
90+
formatter_class=argparse.RawTextHelpFormatter)
91+
parser.add_argument('generate_input',
92+
help='Generate input file path')
93+
parser.add_argument('generate_output',
94+
help='Generate output file path')
95+
parser.add_argument("-v", "--verbose",
96+
dest="verbose", action="store_true",
97+
help="Verbosity in INFO mode")
98+
parser.add_argument("--debug",
99+
dest="debug", action="store_true",
100+
help="Verbosity in DEBUG mode")
101+
parser.add_argument("-c", "--codegen",
102+
dest="debug", action="store_true",
103+
help="Verbosity in DEBUG mode")
104+
105+
args = parser.parse_args()
106+
main_logger = logging.getLogger()
107+
logging.basicConfig()
108+
main_logger.setLevel(logging.DEBUG if args.verbose or args.debug else logging.INFO)
109+
110+
main(args.generate_input, args.generate_output)
111+
112+
113+
if __name__ == "__main__":
114+
generate_main()

0 commit comments

Comments
 (0)