From c53a654ef99d172658678928b2e15cc585e580fd Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 5 Feb 2021 16:14:19 +0800 Subject: [PATCH 01/59] 0.0.1 --- scripts/auto_release/CHANGELOG.md | 5 + scripts/auto_release/PythonSdkLiveTest.yml | 183 ++++++++++ scripts/auto_release/livetest_folder_link.py | 124 +++++++ scripts/auto_release/main.py | 364 +++++++++++++++++++ scripts/auto_release/usage.txt | 10 + scripts/auto_release/version.py | 9 + 6 files changed, 695 insertions(+) create mode 100644 scripts/auto_release/CHANGELOG.md create mode 100644 scripts/auto_release/PythonSdkLiveTest.yml create mode 100644 scripts/auto_release/livetest_folder_link.py create mode 100644 scripts/auto_release/main.py create mode 100644 scripts/auto_release/usage.txt create mode 100644 scripts/auto_release/version.py diff --git a/scripts/auto_release/CHANGELOG.md b/scripts/auto_release/CHANGELOG.md new file mode 100644 index 000000000000..f5aff2942dc7 --- /dev/null +++ b/scripts/auto_release/CHANGELOG.md @@ -0,0 +1,5 @@ +# Release History + +## 1.0.0b1 (data_format) + +* Initial Release diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml new file mode 100644 index 000000000000..8ed983fa6c4f --- /dev/null +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -0,0 +1,183 @@ +# Live Test For Python SDK + +name: PYTHON SDK LIVE TEST RUN $(USER_TARGET) $(USER_LIVE) $(USER_USERNAME) $(Date:yyyyMMdd)$(Rev:.r) + +trigger: + branches: + exclude: + - '*' + + +jobs: +- job: LiveTestPython38 + displayName: Live Test Python 3.8 + timeoutInMinutes: 1500 + strategy: + maxParallel: 5 + pool: + vmImage: 'ubuntu-18.04' + steps: + - task: UsePythonVersion38 + inputs: + versionSpec: '3.8' + addToPath: true + architecture: 'x64' + - bash: | + script_path=$(pwd) + cd .. + git config --global user.email "PythonSdkPipelines" + git config --global user.name "PythonSdkPipelines" + + # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) + git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} + + # create virtual env + python -m venv venv-sdk + source venv-sdk/bin/activate + output_path=$(pwd)/venv-sdk + + # import env variable + + + # create branch + cd azure-sdk-for-python + python /home/vsts/work/1/s/scripts/auto_release/main.py "$(BASE_BRANCH)" "$script_path" "$output_path" + + # + sed -n '6p' filename 打印文件的第6行 + + +# # Whether commit changes +# if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then +# REPO="$(USER_REPO)" +# # Pass username and token so that we can commit changes +# git clone -b $(USER_BRANCH) ${REPO:0:8}$(USER_USERNAME):$(USER_TOKEN)@${REPO:8} +# git clone https://github.com/Azure/azure-cli-extensions.git +# else +# git clone -b $(USER_BRANCH) $(USER_REPO) +# git clone https://github.com/Azure/azure-cli-extensions.git +# fi +# +# python -m venv env +# source env/bin/activate +# pip install azdev +# # git clone https://github.com/Azure/azure-cli-dev-tools.git +# # git clone -b fix https://github.com/qwordy/azure-cli-dev-tools.git +# # pip install -e azure-cli-dev-tools +# pip install pytest-json-report +# pip install pytest-html +# pip install pytest-rerunfailures +# +# azdev setup -c azure-cli -r azure-cli-extensions +# +# # Remove ext- prefix +# FinalTarget="$(Target)" +# if [[ "${FinalTarget}" =~ ^ext-.* ]]; then +# echo "It is an extention" +# echo $(Target) +# FinalTarget=${FinalTarget:4} +# azdev extension add ${FinalTarget} +# fi +# echo ${FinalTarget} +# +# az -v +# # az login --service-principal --username $(APP_ID) --password $(PASSWORD) --tenant $(TENANT_ID) +# az login -u azureclitest@azuresdkteam.onmicrosoft.com -p $(AZURECLITESTPASSWORD) +# az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590 +# +# # Clean resources +# python /home/vsts/work/1/s/scripts/live_test/clean_policy.py +# +# # Whether commit changes +# if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then +# echo "Commit mode" +# azdev test ${FinalTarget} --no-exitfirst -a "-n $(USER_PARALLELISM)" +# azdev test ${FinalTarget} --live --lf --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).report.parallel.json --html=$(Target).report.parallel.html --self-contained-html --reruns 3 --capture=sys" +# else +# echo "Normal mode" +# # Sequential +# azdev test ${FinalTarget} $(USER_LIVE) --mark serial --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).report.sequential.json --html=$(Target).report.sequential.html --self-contained-html --reruns 3 --capture=sys" +# # Parallel +# azdev test ${FinalTarget} $(USER_LIVE) --mark "not serial" --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).report.parallel.json --html=$(Target).report.parallel.html --self-contained-html --reruns 3 --capture=sys" +# fi + +# pwd +# ls +# # Whether commit changes +# if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then +# cd azure-cli +# git status +# git add . +# git commit -m "Upload recording files" +# git push origin $(USER_BRANCH) +# fi +# +# condition: and(succeededOrFailed(), eq(variables.Match, '1')) +# - task: PublishTestResults@2 +# condition: and(succeededOrFailed(), eq(variables.Match, '1')) +# inputs: +# testResultsFiles: '/home/vsts/work/1/test_results.*.xml' +# testRunTitle: 'Azure CLI live test results of $(Target)' +# mergeTestResults: true +# # testResultsFormat: 'JUnit' +# - task: PublishBuildArtifacts@1 +# condition: and(succeededOrFailed(), eq(variables.Match, '1')) +# inputs: +# PathtoPublish: '/home/vsts/work/1/$(Target).report.sequential.json' +# ArtifactName: '$(Target).report.sequential.json' +# publishLocation: 'Container' +# - task: PublishBuildArtifacts@1 +# condition: and(succeededOrFailed(), eq(variables.Match, '1')) +# inputs: +# PathtoPublish: '/home/vsts/work/1/$(Target).report.sequential.html' +# ArtifactName: '$(Target).report.sequential.html' +# publishLocation: 'Container' +# - task: PublishBuildArtifacts@1 +# condition: and(succeededOrFailed(), eq(variables.Match, '1')) +# inputs: +# PathtoPublish: '/home/vsts/work/1/$(Target).report.parallel.json' +# ArtifactName: '$(Target).report.parallel.json' +# publishLocation: 'Container' +# - task: PublishBuildArtifacts@1 +# condition: and(succeededOrFailed(), eq(variables.Match, '1')) +# inputs: +# PathtoPublish: '/home/vsts/work/1/$(Target).report.parallel.html' +# ArtifactName: '$(Target).report.parallel.html' +# publishLocation: 'Container' +# +#- job: SendEmail +# displayName: Send email +# timeoutInMinutes: 180 +# dependsOn: LiveTestPython38 +# condition: succeededOrFailed() +# steps: +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: '3.8' +# addToPath: true +# architecture: 'x64' +# - task: DownloadBuildArtifacts@0 +# inputs: +# buildType: 'current' +# downloadType: 'specific' +# downloadPath: '$(System.ArtifactsDirectory)' +# - bash: | +# echo $(System.ArtifactsDirectory) +# ls $(System.ArtifactsDirectory) +# pwd +# commit_id=`git ls-remote https://github.com/Azure/azure-cli.git HEAD` +# pip install sendgrid +# pip install mysql-connector-python +# pip install requests +# # pip install certifi +# # Send notification +# az -v +# az login -u azureclitest@azuresdkteam.onmicrosoft.com -p $(AZURECLITESTPASSWORD) +# az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590 +# python /home/vsts/work/1/s/scripts/live_test/sendemail.py "$(SENDGRID_KEY)" "$(Build.BuildId)" "$(USER_REPO)" "$(USER_BRANCH)" "$(USER_TARGET)" "$(USER_LIVE)" "$(System.ArtifactsDirectory)" "$(Build.RequestedForEmail)" "$(ACCOUNT_KEY)" "$commit_id" "$(DB_PWD)" +# # python /home/vsts/work/1/s/scripts/live_test/upload.py "$(System.ArtifactsDirectory)" "$(ACCOUNT_KEY)" "$(USER_LIVE)" +# # Clean resources +# if [[ "$(USER_LIVE)" == "--live" && "$(USER_TARGET)" == "" ]]; then +# python /home/vsts/work/1/s/scripts/live_test/clean.py +# fi +# diff --git a/scripts/auto_release/livetest_folder_link.py b/scripts/auto_release/livetest_folder_link.py new file mode 100644 index 000000000000..d8e5b8e5b2f7 --- /dev/null +++ b/scripts/auto_release/livetest_folder_link.py @@ -0,0 +1,124 @@ +# ServiceName:SdkFolder +FOLDER_LINK={ + 'advisor' : 'advisor', + 'devspaces' : 'aks', + 'alertsmanagement' : 'alertsmanagement', + 'apimanagement' : 'apimanagement', + 'appconfiguration' : 'appconfiguration', + 'applicationinsights' : 'applicationinsights', + 'appplatform' : 'appplatform', + 'web' : 'appservice', + 'attestation' : 'attestation', + 'authorization' : 'authorization', + 'automanage' : 'automanage', + 'automation' : 'automation', + 'azurestack' : 'azurestack', + 'baremetalinfrastructure' : 'baremetalinfrastructure', + 'batch' : 'batch', + 'billing' : 'billing', + 'botservice' : 'botservice', + 'cdn' : 'cdn', + 'cognitiveservices' : 'cognitiveservices', + 'commerce' : 'commerce', + 'communication' : 'communication', + 'compute' : 'compute', + 'imagebuilder' : 'compute', + 'vmwarecloudsimple' : 'compute', + 'confluent' : 'confluent', + 'consumption' : 'consumption', + 'containerinstance' : 'containerinstance', + 'containerregistry' : 'containerregistry', + 'containerservice' : 'containerservice', + 'core' : 'core', + 'datalake-nspkg' : 'core', + 'nspkg' : 'core', + 'cosmosdb' : 'cosmos', + 'documentdb' : 'cosmos', + 'costmanagement' : 'costmanagement', + 'customproviders' : 'customproviders', + 'databox' : 'databox', + 'databoxedge' : 'databoxedge', + 'databricks' : 'databricks', + 'datadog' : 'datadog', + 'datafactory' : 'datafactory', + 'datalake-analytics' : 'datalake', + 'datalake-store' : 'datalake', + 'datamigration' : 'datamigration', + 'datashare' : 'datashare', + 'deploymentmanager' : 'deploymentmanager', + 'devtestlabs' : 'devtestlabs', + 'digitaltwins' : 'digitaltwins', + 'edgegateway' : 'edgegateway', + 'eventgrid' : 'eventgrid', + 'eventhub' : 'eventhub', + 'hanaonazure' : 'hanaonazure', + 'hdinsight' : 'hdinsight', + 'healthcareapis' : 'healthcareapis', + 'hybridcompute' : 'hybridcompute', + 'hybridkubernetes' : 'hybridkubernetes', + 'iotcentral' : 'iothub', + 'iothub' : 'iothub', + 'iothubprovisioningservices' : 'iothub', + 'keyvault' : 'keyvault', + 'kubernetesconfiguration' : 'kubernetesconfiguration', + 'kusto' : 'kusto', + 'labservices' : 'labservices', + 'loganalytics' : 'loganalytics', + 'logic' : 'logic', + 'machinelearningcompute' : 'machinelearning', + 'machinelearningservices' : 'machinelearning', + 'maintenance' : 'maintenance', + 'managedservices' : 'managedservices', + 'managementgroups' : 'managementgroups', + 'managementpartner' : 'managementpartner', + 'maps' : 'maps', + 'marketplaceordering' : 'marketplaceordering', + 'media' : 'media', + 'mixedreality' : 'mixedreality', + 'monitor' : 'monitor', + 'netapp' : 'netapp', + 'dns' : 'network', + 'frontdoor' : 'network', + 'network' : 'network', + 'privatedns' : 'network', + 'notificationhubs' : 'notificationhubs', + 'operationsmanagement' : 'operationsmanagement', + 'peering' : 'peering', + 'policyinsights' : 'policyinsights', + 'powerbidedicated' : 'powerbidedicated', + 'powerbiembedded' : 'powerbiembedded', + 'rdbms' : 'rdbms', + 'recoveryservices' : 'recoveryservices', + 'recoveryservicesbackup' : 'recoveryservices', + 'redhatopenshift' : 'redhatopenshift', + 'redis' : 'redis', + 'regionmove' : 'regionmove', + 'relay' : 'relay', + 'reservations' : 'reservations', + 'resourcemover' : 'resourcemover', + 'msi' : 'resources', + 'resource' : 'resources', + 'resourcegraph' : 'resources', + 'scheduler' : 'scheduler', + 'search' : 'search', + 'security' : 'security', + 'securityinsight' : 'securityinsight', + 'serialconsole' : 'serialconsole', + 'servermanager' : 'servermanager', + 'servicebus' : 'servicebus', + 'servicefabric' : 'servicefabric', + 'signalr' : 'signalr', + 'sql' : 'sql', + 'sqlvirtualmachine' : 'sql', + 'storage' : 'storage', + 'storagecache' : 'storage', + 'storageimportexport' : 'storage', + 'storagesync' : 'storage', + 'streamanalytics' : 'streamanalytics', + 'subscription' : 'subscription', + 'support' : 'support', + 'synapse' : 'synapse', + 'timeseriesinsights' : 'timeseriesinsights', + 'trafficmanager' : 'trafficmanager', + 'workloadmonitor' : 'workloadmonitor', +} diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py new file mode 100644 index 000000000000..879e8aea066c --- /dev/null +++ b/scripts/auto_release/main.py @@ -0,0 +1,364 @@ +import os +import sys +import subprocess as sp +import time +import re +import argparse + +SERVICE_NAME = 'servicename' +SDK_FOLDER = 'servicename' +TRACK = '1' +VERSION_NEW = '0.0.0' +VERSION_OLD = '0.0.0' +VERSION_LAST_RELEASE = '0.0.0' +BRANCH_BASE = '' +OUT_PATH = '' +NEW_BRANCH = '' + + +def my_print(cmd): + print(f'({SERVICE_NAME})==' + cmd + ' ==\n') + + +def print_exec(cmd): + my_print(cmd) + sp.call(cmd, shell=True) + + +def print_exec_output(cmd): + my_print(cmd) + return sp.getoutput(cmd).split('\n') + + +def print_check(cmd): + my_print(cmd) + sp.check_call(cmd, shell=True) + + +def find_report_name(result): + pattern = 'written to' + merged = 'merged_report' + for line in result: + idx = line.find(pattern) + idx1 = line.find(merged) + if idx > 0 and idx1 > 0: + return line[idx + len(pattern):] + + for line in result: + idx = line.find(pattern) + if idx > 0: + return line[idx + len(pattern):] + + return '' + + +def get_version(report): + global VERSION_LAST_RELEASE + pattern = 'code_reports/' + idx1 = report.find(pattern) + idx2 = report.find('/', idx1 + len(pattern)) + VERSION_LAST_RELEASE = report[idx1 + len(pattern):idx2] + + +def create_changelog_content(): + result1 = print_exec_output(f'python -m packaging_tools.code_report --last-pypi azure-mgmt-{SERVICE_NAME}') + report1 = find_report_name(result1) + result2 = print_exec_output(f'python -m packaging_tools.code_report azure-mgmt-{SERVICE_NAME}') + report2 = find_report_name(result2) + result = print_exec_output(f'python -m packaging_tools.change_log {report1} {report2}') + if len(result) > 0: + add_content = result[1:] + get_version(report1) + else: + add_content = [] + + return add_content + + +def edit_version(add_content): + global VERSION_NEW, VERSION_OLD + flag = [False, False, False] # breaking, feature, bugfix + for line in add_content: + if line.find('**Breaking changes**') > -1: + flag[0] = True + break + elif line.find('**Features**') > -1: + flag[1] = True + elif line.find('**Bugfixes**') > -1: + flag[2] = True + + path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/azure/mgmt/{SERVICE_NAME}' + file_name = 'version.py' if TRACK == '1' else '_version.py' + with open(f'{path}/{file_name}', 'r') as file_in: + list_in = file_in.readlines() + i = 0 + while i < len(list_in): + result = re.search('\".+\"', list_in[i]) + if result: + idx = result.span() + VERSION_OLD = list_in[i][idx[0] + 1: idx[1] - 1] + break + i = i + 1 + + num = VERSION_OLD.split('.') + if TRACK == '1' and num[0] == '0': + VERSION_NEW = f'0.{str(int(num[1]) + 1)}.0' + elif TRACK == '2' and VERSION_OLD.find('b') > -1: + lastnum = num[2].split('b') + lastnum[1] = str(int(lastnum[1]) + 1) + VERSION_NEW = f'{num[0]}.{num[1]}.{lastnum[0]}b{lastnum[1]}' + elif flag[0]: + VERSION_NEW = f'{int(num[0]) + 1}.0.0' + elif flag[1]: + VERSION_NEW = f'{num[0]}.{int(num[1]) + 1}.0' + elif flag[2]: + VERSION_NEW = f'{num[0]}.{num[1]}.{int(num[2]) + 1}' + + list_in[i] = f'VERSION = "{VERSION_NEW}"\n' + with open(f'{path}/{file_name}', 'w') as file_out: + file_out.writelines(list_in) + + +def edit_changelog(add_content): + path = f'/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' + with open(f'{path}/CHANGELOG.md', 'r') as file_in: + list_in = file_in.readlines() + list_out = [list_in[0], '\n'] + date = time.localtime(time.time()) + list_out.append('## {} ({}-{:02d}-{:02d})\n\n'.format(VERSION_NEW, date.tm_year, date.tm_mon, date.tm_mday)) + for line in add_content: + list_out.append(line + '\n') + list_out.extend(list_in[1:]) + with open(f'{path}/CHANGELOG.md', 'w') as file_out: + file_out.writelines(list_out) + + +def print_changelog(add_content): + for line in add_content: + my_print('[CHANGELOG] ' + line) + + +def edit_file_setup(): + path = f'/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' + with open(f'{path}/setup.py', 'r') as file_in: + list_in = file_in.readlines() + for i in range(0, len(list_in)): + list_in[i] = list_in[i].replace('msrestazure>=0.4.32,<2.0.0', 'azure-mgmt-core>=1.2.0,<2.0.0') + with open(f'{path}/setup.py', 'w') as file_out: + file_out.writelines(list_in) + + +def edit_file_readme(): + path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' + # read and edit usage part + with open(f'{SCRIPT_PATH}/usage.txt', 'r') as file_in: + usage = file_in.readlines() + for i in range(0, len(usage)): + usage[i] = re.sub('MyService', SERVICE_NAME.capitalize(), usage[i], flags=re.IGNORECASE) + # edit README + with open(f'{path}/README.md') as file_in: + list_in = file_in.readlines() + i = 1 + list_out = [list_in[0]] + while i < len(list_in): + if list_in[i].find('## Usage') > -1: + break + list_out.append(list_in[i]) + i = i + 1 + list_out.extend(usage) + flag = False + while i < len(list_in): + if list_in[i].find('## Provide Feedback') > -1: + flag = True + if flag: + list_out.append(list_in[i]) + i = i + 1 + with open(f'{path}/README.md', 'w') as file_out: + file_out.writelines(list_out) + + +def edit_first_release(): + global VERSION_NEW + VERSION_NEW = '1.0.0b1' + # edit version.py + path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/azure/mgmt/{SERVICE_NAME}' + file_name = 'version.py' if TRACK == '1' else '_version.py' + print_check(f'cp {SCRIPT_PATH}/version.py {path}/{file_name}') + + # edit CHANGELOG.md + with open(f'{SCRIPT_PATH}/CHANGELOG.md', 'r') as file_in: + content = file_in.readlines() + + date = time.localtime(time.time()) + data_format = '{}-{:02d}-{:02d}'.format(date.tm_year, date.tm_mon, date.tm_mday) + for i in range(0, len(content)): + content[i] = content[i].replace('data_format', data_format) + with open(f'{path}/CHANGELOG.md', 'w') as file_out: + file_out.writelines(content) + + +def edit_file(): + from pypi_tools.pypi import PyPIClient + client = PyPIClient() + try: + client.get_ordered_versions(f'azure-mgmt-{SERVICE_NAME}') + except: + print_changelog(['* Initial Release']) + edit_first_release() + my_print(f'CHANGELOG and version(new:{VERSION_NEW}) generate successfully. It is first release') + else: + add_content = create_changelog_content() + if len(add_content) == 0: + raise Exception('changelog and version generate failed, please do it manually') + else: + print_changelog(add_content) + edit_version(add_content) + edit_changelog(add_content) + if TRACK == '2': + edit_file_setup() + edit_file_readme() + my_print(f'CHANGELOG and version(new:{VERSION_NEW}) generate successfully, please check it(compare with ' + f'{VERSION_LAST_RELEASE}[https://pypi.org/pypi/azure-mgmt-{SERVICE_NAME}/{VERSION_LAST_RELEASE}])') + + +def build_wheel(): + path = os.getcwd() + setup_path = f'{path}/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' + print_check(f'cd {setup_path} && python setup.py bdist_wheel') + print_check('cd path') + + +def test_env_init(): + print_check(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') + file = f'{SCRIPT_PATH}/livetest_package_{SERVICE_NAME}_track{TRACK}.txt' + if os.path.exists(file): + print_exec(f'pip install -r {file}') + + +def run_live_test(): + test_env_init() + print_exec(f'python scripts/dev_setup.py -p azure-mgmt-{SERVICE_NAME}') + # run live test + print_exec(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') + my_print('live test run done !!!') + + +def del_useless_file(): + if TRACK == '1': + return + + path = f'{os.getcwd()}/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/azure/mgmt/{SERVICE_NAME}' + for folder in os.listdir(path): + if os.path.isdir(f'{path}/{folder}') and os.path.exists(f'{path}/{folder}/_version.py'): + print_exec(f'rm -rf {path}/{folder}/_version.py') + + +def commit_test(): + print_exec('git add sdk/') + print_exec('git commit -m \"test"') + print_exec('git push -f origin HEAD') + my_print(f'== {SERVICE_NAME}(track{TRACK}) Automatic Release live test done !!! ==') + + +def init_env(): + print_exec(f'python scripts/dev_setup.py -p azure-mgmt-{SERVICE_NAME}') + + +def check_pprint_name(): + path = f'{os.getcwd()}/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' + pprint_name = SERVICE_NAME.capitalize() + for file in os.listdir(path): + file_path = f'{path}/{file}' + if os.path.isfile(file_path): + with open(file_path, 'r') as file_in: + list_in = file_in.readlines() + for i in range(0, len(list_in)): + list_in[i] = list_in[i].replace('MyService', pprint_name) + with open(file_path, 'w') as file_out: + file_out.writelines(list_in) + my_print(f'== replace\"MyService\" with \"{pprint_name}\" successfully ==') + + +def judge_sdk_folder(): + global SDK_FOLDER, TRACK + from livetest_folder_link import FOLDER_LINK + + SDK_FOLDER = FOLDER_LINK[SERVICE_NAME] if SERVICE_NAME in FOLDER_LINK else SERVICE_NAME + sdk_path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' + if not os.path.exists(sdk_path): + raise Exception(f'{sdk_path} does not exist, please update livetest_folder_link.py') + + # additional rule to judge track1 or track2 + if os.path.exists('swagger_to_sdk_config_autorest.json'): + with open('swagger_to_sdk_config_autorest.json', 'r') as file_in: + content = file_in.readlines() + for line in content: + if line.find('azure-sdk-for-python-track2') > 0: + TRACK = '2' + break + + +def git_remote_add(): + global TRACK, NEW_BRANCH + # init git + print_exec('git remote add autosdk https://github.com/AzureSDKAutomation/azure-sdk-for-python.git') + print_check(f'get fetch autosdk {BRANCH_BASE}') + print_check(f'get checkout autosdk/{BRANCH_BASE}') + + +def create_branch(): + global NEW_BRANCH + # create new branch + date = time.localtime(time.time()) + NEW_BRANCH = 't{}-{}-{}-{:02d}-{:02d}'.format(TRACK, SERVICE_NAME, date.tm_year, date.tm_mon, date.tm_mday) + print_exec(f'git checkout -b {NEW_BRANCH}') + + +def commit_file(): + print_exec('git add sdk/') + print_exec('git commit -m \"version,CHANGELOG\"') + print_exec('git push -f origin HEAD') + my_print(f'== {SERVICE_NAME}(track{TRACK}) Automatic Release file-edit done !!! ==') + + +def main(): + git_remote_add() + judge_sdk_folder() + create_branch() + init_env() + del_useless_file() + edit_file() + check_pprint_name() + commit_file() + run_live_test() + build_wheel() + commit_test() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Auto release', + formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument("branch", help="branch name") + parser.add_argument("script_path", help="path where the script is") + parser.add_argument("out_path", help="path where the output is") + args = parser.parse_args() + + BRANCH_BASE = args.branch.replace('AzureSDKAutomation:', '') + SCRIPT_PATH = args.script_path + OUT_PATH = args.out_path + + # extract info + sys.path.append(OUT_PATH) + TRACK = '2' if BRANCH_BASE.find('track2_') > -1 else '1' + SERVICE_NAME = BRANCH_BASE.replace('sdkAuto/', '').replace('sdkAutomation/', '').replace('track2_', '').replace( + 'azure-mgmt-', '') + try: + main() + except Exception as e: + my_print(e) + except sp.CalledProcessError as e: + my_print(e) + else: + print_exec(f'touch {OUT_PATH}/output.txt') diff --git a/scripts/auto_release/usage.txt b/scripts/auto_release/usage.txt new file mode 100644 index 000000000000..ed8eafd4d9d5 --- /dev/null +++ b/scripts/auto_release/usage.txt @@ -0,0 +1,10 @@ +# Usage + + +To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) + + + +For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/) +Code samples for this package can be found at [MyService Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. +Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) diff --git a/scripts/auto_release/version.py b/scripts/auto_release/version.py new file mode 100644 index 000000000000..e5754a47ce68 --- /dev/null +++ b/scripts/auto_release/version.py @@ -0,0 +1,9 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# Code generated by Microsoft (R) AutoRest Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is regenerated. +# -------------------------------------------------------------------------- + +VERSION = "1.0.0b1" From 2b707aeb4c0fc9f62286e5513cefd76616a970fc Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 5 Feb 2021 19:00:26 +0800 Subject: [PATCH 02/59] 0.0.2 --- scripts/auto_release/PythonSdkLiveTest.yml | 168 ++++----------------- scripts/auto_release/livetest_package.txt | 2 + scripts/auto_release/main.py | 36 +++-- scripts/auto_release/pypi.py | 56 +++++++ 4 files changed, 110 insertions(+), 152 deletions(-) create mode 100644 scripts/auto_release/livetest_package.txt create mode 100644 scripts/auto_release/pypi.py diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 8ed983fa6c4f..f16cb69cb992 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -37,147 +37,39 @@ jobs: output_path=$(pwd)/venv-sdk # import env variable + export AZURE_TEST_RUN_LIVE=true + export TENANT_ID=$(ENV_TENANT_ID) + export CLIENT_ID=$(ENV_CLIENT_ID) + export CLIENT_SECRET=$(ENV_CLIENT_SECRET) + export SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) + export AZURE_TENANT_ID=$(ENV_TENANT_ID) + export AZURE_CLIENT_ID=$(ENV_CLIENT_ID) + export AZURE_CLIENT_SECRET=$(ENV_CLIENT_SECRET) + export AZURE_SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) - - # create branch + # run cd azure-sdk-for-python python /home/vsts/work/1/s/scripts/auto_release/main.py "$(BASE_BRANCH)" "$script_path" "$output_path" - # - sed -n '6p' filename 打印文件的第6行 - + # judge + if [ ! -f "$output_path/output.txt" ]; then + exit 1 + fi -# # Whether commit changes -# if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then -# REPO="$(USER_REPO)" -# # Pass username and token so that we can commit changes -# git clone -b $(USER_BRANCH) ${REPO:0:8}$(USER_USERNAME):$(USER_TOKEN)@${REPO:8} -# git clone https://github.com/Azure/azure-cli-extensions.git -# else -# git clone -b $(USER_BRANCH) $(USER_REPO) -# git clone https://github.com/Azure/azure-cli-extensions.git -# fi -# -# python -m venv env -# source env/bin/activate -# pip install azdev -# # git clone https://github.com/Azure/azure-cli-dev-tools.git -# # git clone -b fix https://github.com/qwordy/azure-cli-dev-tools.git -# # pip install -e azure-cli-dev-tools -# pip install pytest-json-report -# pip install pytest-html -# pip install pytest-rerunfailures -# -# azdev setup -c azure-cli -r azure-cli-extensions -# -# # Remove ext- prefix -# FinalTarget="$(Target)" -# if [[ "${FinalTarget}" =~ ^ext-.* ]]; then -# echo "It is an extention" -# echo $(Target) -# FinalTarget=${FinalTarget:4} -# azdev extension add ${FinalTarget} -# fi -# echo ${FinalTarget} -# -# az -v -# # az login --service-principal --username $(APP_ID) --password $(PASSWORD) --tenant $(TENANT_ID) -# az login -u azureclitest@azuresdkteam.onmicrosoft.com -p $(AZURECLITESTPASSWORD) -# az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590 -# -# # Clean resources -# python /home/vsts/work/1/s/scripts/live_test/clean_policy.py -# -# # Whether commit changes -# if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then -# echo "Commit mode" -# azdev test ${FinalTarget} --no-exitfirst -a "-n $(USER_PARALLELISM)" -# azdev test ${FinalTarget} --live --lf --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).report.parallel.json --html=$(Target).report.parallel.html --self-contained-html --reruns 3 --capture=sys" -# else -# echo "Normal mode" -# # Sequential -# azdev test ${FinalTarget} $(USER_LIVE) --mark serial --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).report.sequential.json --html=$(Target).report.sequential.html --self-contained-html --reruns 3 --capture=sys" -# # Parallel -# azdev test ${FinalTarget} $(USER_LIVE) --mark "not serial" --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).report.parallel.json --html=$(Target).report.parallel.html --self-contained-html --reruns 3 --capture=sys" -# fi + # create PR + new_branch = `sed -n '1p' $output_path/output.txt` + target_branch = `sed -n '2p' $output_path/output.txt` + curl \ + --user "anything:$(USR_TOKEN)" \ + -d "{\ + \"title\":\"[AutoRelease_Python] $new_branch\",\ + \"body\":\"Please Add the link issue\",\ + \"head\":\"$(USR_NAME):$new_branch\",\ + \"base\":\"$target_branch\"\ + }" \ + $(REPO)/pulls -# pwd -# ls -# # Whether commit changes -# if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then -# cd azure-cli -# git status -# git add . -# git commit -m "Upload recording files" -# git push origin $(USER_BRANCH) -# fi -# -# condition: and(succeededOrFailed(), eq(variables.Match, '1')) -# - task: PublishTestResults@2 -# condition: and(succeededOrFailed(), eq(variables.Match, '1')) -# inputs: -# testResultsFiles: '/home/vsts/work/1/test_results.*.xml' -# testRunTitle: 'Azure CLI live test results of $(Target)' -# mergeTestResults: true -# # testResultsFormat: 'JUnit' -# - task: PublishBuildArtifacts@1 -# condition: and(succeededOrFailed(), eq(variables.Match, '1')) -# inputs: -# PathtoPublish: '/home/vsts/work/1/$(Target).report.sequential.json' -# ArtifactName: '$(Target).report.sequential.json' -# publishLocation: 'Container' -# - task: PublishBuildArtifacts@1 -# condition: and(succeededOrFailed(), eq(variables.Match, '1')) -# inputs: -# PathtoPublish: '/home/vsts/work/1/$(Target).report.sequential.html' -# ArtifactName: '$(Target).report.sequential.html' -# publishLocation: 'Container' -# - task: PublishBuildArtifacts@1 -# condition: and(succeededOrFailed(), eq(variables.Match, '1')) -# inputs: -# PathtoPublish: '/home/vsts/work/1/$(Target).report.parallel.json' -# ArtifactName: '$(Target).report.parallel.json' -# publishLocation: 'Container' -# - task: PublishBuildArtifacts@1 -# condition: and(succeededOrFailed(), eq(variables.Match, '1')) -# inputs: -# PathtoPublish: '/home/vsts/work/1/$(Target).report.parallel.html' -# ArtifactName: '$(Target).report.parallel.html' -# publishLocation: 'Container' -# -#- job: SendEmail -# displayName: Send email -# timeoutInMinutes: 180 -# dependsOn: LiveTestPython38 -# condition: succeededOrFailed() -# steps: -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: '3.8' -# addToPath: true -# architecture: 'x64' -# - task: DownloadBuildArtifacts@0 -# inputs: -# buildType: 'current' -# downloadType: 'specific' -# downloadPath: '$(System.ArtifactsDirectory)' -# - bash: | -# echo $(System.ArtifactsDirectory) -# ls $(System.ArtifactsDirectory) -# pwd -# commit_id=`git ls-remote https://github.com/Azure/azure-cli.git HEAD` -# pip install sendgrid -# pip install mysql-connector-python -# pip install requests -# # pip install certifi -# # Send notification -# az -v -# az login -u azureclitest@azuresdkteam.onmicrosoft.com -p $(AZURECLITESTPASSWORD) -# az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590 -# python /home/vsts/work/1/s/scripts/live_test/sendemail.py "$(SENDGRID_KEY)" "$(Build.BuildId)" "$(USER_REPO)" "$(USER_BRANCH)" "$(USER_TARGET)" "$(USER_LIVE)" "$(System.ArtifactsDirectory)" "$(Build.RequestedForEmail)" "$(ACCOUNT_KEY)" "$commit_id" "$(DB_PWD)" -# # python /home/vsts/work/1/s/scripts/live_test/upload.py "$(System.ArtifactsDirectory)" "$(ACCOUNT_KEY)" "$(USER_LIVE)" -# # Clean resources -# if [[ "$(USER_LIVE)" == "--live" && "$(USER_TARGET)" == "" ]]; then -# python /home/vsts/work/1/s/scripts/live_test/clean.py -# fi -# + # if test failed, still push and crete PR. But pipeline failed to remind user + if [ -f "$output_path/live_test_fail.txt" ]; then + exit 1 + fi diff --git a/scripts/auto_release/livetest_package.txt b/scripts/auto_release/livetest_package.txt new file mode 100644 index 000000000000..74fd8517c3fa --- /dev/null +++ b/scripts/auto_release/livetest_package.txt @@ -0,0 +1,2 @@ +azure-identity +python-dotenv==0.15.0 \ No newline at end of file diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 879e8aea066c..d63b5e9956dc 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -120,7 +120,7 @@ def edit_version(add_content): def edit_changelog(add_content): - path = f'/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' + path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' with open(f'{path}/CHANGELOG.md', 'r') as file_in: list_in = file_in.readlines() list_out = [list_in[0], '\n'] @@ -135,11 +135,11 @@ def edit_changelog(add_content): def print_changelog(add_content): for line in add_content: - my_print('[CHANGELOG] ' + line) + print('[CHANGELOG] ' + line) def edit_file_setup(): - path = f'/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' + path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' with open(f'{path}/setup.py', 'r') as file_in: list_in = file_in.readlines() for i in range(0, len(list_in)): @@ -198,7 +198,7 @@ def edit_first_release(): def edit_file(): - from pypi_tools.pypi import PyPIClient + from pypi import PyPIClient client = PyPIClient() try: client.get_ordered_versions(f'azure-mgmt-{SERVICE_NAME}') @@ -225,7 +225,7 @@ def build_wheel(): path = os.getcwd() setup_path = f'{path}/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' print_check(f'cd {setup_path} && python setup.py bdist_wheel') - print_check('cd path') + print_check(f'cd {path}') def test_env_init(): @@ -239,8 +239,13 @@ def run_live_test(): test_env_init() print_exec(f'python scripts/dev_setup.py -p azure-mgmt-{SERVICE_NAME}') # run live test - print_exec(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') - my_print('live test run done !!!') + try: + print_check(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') + except: + my_print('some test failed, please fix it locally') + print_check(f'touch {OUT_PATH}/live_test_fail.txt') + else: + my_print('live test run done !!!') def del_useless_file(): @@ -301,16 +306,18 @@ def judge_sdk_folder(): def git_remote_add(): global TRACK, NEW_BRANCH # init git + print_exec('git checkout . && git clean -fd && git reset --hard HEAD ') print_exec('git remote add autosdk https://github.com/AzureSDKAutomation/azure-sdk-for-python.git') - print_check(f'get fetch autosdk {BRANCH_BASE}') - print_check(f'get checkout autosdk/{BRANCH_BASE}') + print_check(f'git fetch autosdk {BRANCH_BASE}') + print_check(f'git checkout autosdk/{BRANCH_BASE}') def create_branch(): global NEW_BRANCH # create new branch - date = time.localtime(time.time()) - NEW_BRANCH = 't{}-{}-{}-{:02d}-{:02d}'.format(TRACK, SERVICE_NAME, date.tm_year, date.tm_mon, date.tm_mday) + t = time.time() + d = time.localtime(t) + NEW_BRANCH = 't{}-{}-{}-{:02d}-{:02d}-{}'.format(TRACK, SERVICE_NAME, d.tm_year, d.tm_mon, d.tm_mday, str(t)[-6:]) print_exec(f'git checkout -b {NEW_BRANCH}') @@ -329,10 +336,10 @@ def main(): del_useless_file() edit_file() check_pprint_name() - commit_file() + # commit_file() run_live_test() build_wheel() - commit_test() + # commit_test() if __name__ == '__main__': @@ -361,4 +368,5 @@ def main(): except sp.CalledProcessError as e: my_print(e) else: - print_exec(f'touch {OUT_PATH}/output.txt') + with open(f'{OUT_PATH}/output.txt', 'w') as file_out: + file_out.writelines([f'{NEW_BRANCH}\n', "master" if TRACK == '2' else 'release/v3']) diff --git a/scripts/auto_release/pypi.py b/scripts/auto_release/pypi.py new file mode 100644 index 000000000000..9eb28db8c0fe --- /dev/null +++ b/scripts/auto_release/pypi.py @@ -0,0 +1,56 @@ +from packaging.version import parse as Version + +import requests + +def get_pypi_xmlrpc_client(): + """This is actually deprecated client. + """ + import xmlrpc.client + return xmlrpc.client.ServerProxy("https://pypi.python.org/pypi", use_datetime=True) + +class PyPIClient: + def __init__(self, host="https://pypi.org"): + self._host = host + self._session = requests.Session() + + def project(self, package_name): + response = self._session.get( + "{host}/pypi/{project_name}/json".format( + host=self._host, + project_name=package_name + ) + ) + response.raise_for_status() + return response.json() + + def project_release(self, package_name, version): + response = self._session.get( + "{host}/pypi/{project_name}/{version}/json".format( + host=self._host, + project_name=package_name, + version=version + ) + ) + response.raise_for_status() + return response.json() + + def get_ordered_versions(self, package_name): + project = self.project(package_name) + versions = [ + Version(package_version) + for package_version + in project["releases"].keys() + ] + versions.sort() + return versions + + def get_relevant_versions(self, package_name): + """Return a tuple: (latest release, latest stable) + If there are different, it means the latest is not a stable + """ + versions = self.get_ordered_versions(package_name) + pre_releases = [version for version in versions if not version.is_prerelease] + return ( + versions[-1], + pre_releases[-1] + ) \ No newline at end of file From e1b1b5ac7633f431893204af17cd89beba5c8f89 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 8 Feb 2021 16:51:20 +0800 Subject: [PATCH 03/59] 0.0.3 --- scripts/auto_release/PythonSdkLiveTest.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index f16cb69cb992..8dfe2cfba753 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -1,6 +1,6 @@ # Live Test For Python SDK -name: PYTHON SDK LIVE TEST RUN $(USER_TARGET) $(USER_LIVE) $(USER_USERNAME) $(Date:yyyyMMdd)$(Rev:.r) +name: PYTHON SDK AUTO RELEASE RUN $(BASE_BRANCH) $(Date:yyyyMMdd)$(Rev:.r) trigger: branches: @@ -53,6 +53,7 @@ jobs: # judge if [ ! -f "$output_path/output.txt" ]; then + echo "output.txt does not exist, pipeline fail!!!" exit 1 fi @@ -69,7 +70,10 @@ jobs: }" \ $(REPO)/pulls + echo "\'[AutoRelease_Python] $new_branch \' has been created!!!" + # if test failed, still push and crete PR. But pipeline failed to remind user if [ -f "$output_path/live_test_fail.txt" ]; then + echo "please fix failed test!!! \'[AutoRelease_Python] $new_branch \' has been created!!! " exit 1 fi From 4dbbb1b378c5c1f51f71e78db25e24a513636080 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 8 Feb 2021 17:59:28 +0800 Subject: [PATCH 04/59] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 8dfe2cfba753..ae96978ae387 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -17,7 +17,7 @@ jobs: pool: vmImage: 'ubuntu-18.04' steps: - - task: UsePythonVersion38 + - task: UsePythonVersion@0 inputs: versionSpec: '3.8' addToPath: true From 639e1bd6e195a890e2950b1bb2de3c830b5ee3b5 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 13:49:59 +0800 Subject: [PATCH 05/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index ae96978ae387..72bb384f37dd 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -24,18 +24,23 @@ jobs: architecture: 'x64' - bash: | script_path=$(pwd) + echo 1 $(pwd) cd .. + echo 2 $(pwd) git config --global user.email "PythonSdkPipelines" git config --global user.name "PythonSdkPipelines" # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) - git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} + echo | git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} + echo | ls # create virtual env python -m venv venv-sdk source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk + echo | ls + # import env variable export AZURE_TEST_RUN_LIVE=true export TENANT_ID=$(ENV_TENANT_ID) From d281defc2362d9388a7cea4d8b590434eab07109 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 14:04:09 +0800 Subject: [PATCH 06/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 72bb384f37dd..1567834ded63 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -24,15 +24,17 @@ jobs: architecture: 'x64' - bash: | script_path=$(pwd) - echo 1 $(pwd) + echo 111 $(pwd) cd .. - echo 2 $(pwd) + echo 222 $(pwd) git config --global user.email "PythonSdkPipelines" git config --global user.name "PythonSdkPipelines" # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) - echo | git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} + echo 333 + echo | git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} . echo | ls + echo 444 # create virtual env python -m venv venv-sdk @@ -40,6 +42,7 @@ jobs: output_path=$(pwd)/venv-sdk echo | ls + echo 555 # import env variable export AZURE_TEST_RUN_LIVE=true From 444dc0f847950803f20fbb6a6da5fb3b493f17de Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 14:19:45 +0800 Subject: [PATCH 07/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 1567834ded63..45dfac0f1ecf 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -25,14 +25,18 @@ jobs: - bash: | script_path=$(pwd) echo 111 $(pwd) + echo | ls cd .. echo 222 $(pwd) + echo | ls git config --global user.email "PythonSdkPipelines" git config --global user.name "PythonSdkPipelines" # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) echo 333 - echo | git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} . + mkdir temp_folder + echo | git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder + cd temp_folder echo | ls echo 444 From 52e456f8a8695b256abaf8c3b9b946059a16156e Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 15:06:07 +0800 Subject: [PATCH 08/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 45dfac0f1ecf..a1f454a61f5e 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -35,6 +35,7 @@ jobs: # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) echo 333 mkdir temp_folder + echo ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder echo | git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder cd temp_folder echo | ls From c216a523e925afb3272601509215c0e5fa25e34d Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 15:14:02 +0800 Subject: [PATCH 09/59] delete unnecessary print --- scripts/auto_release/PythonSdkLiveTest.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index a1f454a61f5e..4fc5411d1960 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -24,31 +24,20 @@ jobs: architecture: 'x64' - bash: | script_path=$(pwd) - echo 111 $(pwd) - echo | ls cd .. - echo 222 $(pwd) - echo | ls git config --global user.email "PythonSdkPipelines" git config --global user.name "PythonSdkPipelines" # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) - echo 333 mkdir temp_folder - echo ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder echo | git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder cd temp_folder - echo | ls - echo 444 # create virtual env python -m venv venv-sdk source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk - echo | ls - echo 555 - # import env variable export AZURE_TEST_RUN_LIVE=true export TENANT_ID=$(ENV_TENANT_ID) From 55a51fa228d5646e555646b8d5140be46614cce8 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 15:19:12 +0800 Subject: [PATCH 10/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 4fc5411d1960..ec0e42b2750d 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -51,6 +51,8 @@ jobs: # run cd azure-sdk-for-python + echo 1111111111111111111111 + echo python /home/vsts/work/1/s/scripts/auto_release/main.py "$(BASE_BRANCH)" "$script_path" "$output_path" python /home/vsts/work/1/s/scripts/auto_release/main.py "$(BASE_BRANCH)" "$script_path" "$output_path" # judge From 5e33734fc9ddf3c146e1e4556df6bf3ce7ce55a5 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 15:29:27 +0800 Subject: [PATCH 11/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index ec0e42b2750d..86b6de767dc0 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -30,7 +30,7 @@ jobs: # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) mkdir temp_folder - echo | git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder + git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder cd temp_folder # create virtual env @@ -52,8 +52,10 @@ jobs: # run cd azure-sdk-for-python echo 1111111111111111111111 - echo python /home/vsts/work/1/s/scripts/auto_release/main.py "$(BASE_BRANCH)" "$script_path" "$output_path" - python /home/vsts/work/1/s/scripts/auto_release/main.py "$(BASE_BRANCH)" "$script_path" "$output_path" + echo $BASE_BRANCH + echo python /home/vsts/work/1/s/scripts/auto_release/main.py $(BASE_BRANCH) "$script_path" "$output_path" + echo python /home/vsts/work/1/s/scripts/auto_release/main.py $BASE_BRANCH "$script_path" "$output_path" + python /home/vsts/work/1/s/scripts/auto_release/main.py $(BASE_BRANCH) "$script_path" "$output_path" # judge if [ ! -f "$output_path/output.txt" ]; then From 4925fe31ec6616d0d3e15117fa778e8639fb813b Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 15:48:46 +0800 Subject: [PATCH 12/59] pip install packaging --- scripts/auto_release/PythonSdkLiveTest.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 86b6de767dc0..b7377283895f 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -37,6 +37,7 @@ jobs: python -m venv venv-sdk source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk + pip install packaging # import env variable export AZURE_TEST_RUN_LIVE=true @@ -51,10 +52,6 @@ jobs: # run cd azure-sdk-for-python - echo 1111111111111111111111 - echo $BASE_BRANCH - echo python /home/vsts/work/1/s/scripts/auto_release/main.py $(BASE_BRANCH) "$script_path" "$output_path" - echo python /home/vsts/work/1/s/scripts/auto_release/main.py $BASE_BRANCH "$script_path" "$output_path" python /home/vsts/work/1/s/scripts/auto_release/main.py $(BASE_BRANCH) "$script_path" "$output_path" # judge From 076ae22fc4b093a4948f3b814073f2c2181cfc98 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 16:03:17 +0800 Subject: [PATCH 13/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index b7377283895f..74dd1a040206 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -37,7 +37,8 @@ jobs: python -m venv venv-sdk source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk - pip install packaging + echo | pip install packaging + echo | (pip list | grep packaging) # import env variable export AZURE_TEST_RUN_LIVE=true From 02b8a70e7af211e6d41a63a9c3262f0591b6e0ef Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 16:17:49 +0800 Subject: [PATCH 14/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 74dd1a040206..3bc731c49a74 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -32,6 +32,8 @@ jobs: mkdir temp_folder git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder cd temp_folder + echo 0000000000 + echo | ls # create virtual env python -m venv venv-sdk @@ -52,6 +54,8 @@ jobs: export AZURE_SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) # run + echo 11111111111 + echo | ls cd azure-sdk-for-python python /home/vsts/work/1/s/scripts/auto_release/main.py $(BASE_BRANCH) "$script_path" "$output_path" From d660911411be49cfd2ce98fd13b458ccc4189630 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 16:41:46 +0800 Subject: [PATCH 15/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 3bc731c49a74..771843733427 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -29,9 +29,8 @@ jobs: git config --global user.name "PythonSdkPipelines" # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) - mkdir temp_folder - git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/temp_folder - cd temp_folder + mkdir azure-sdk-for-python + git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/azure-sdk-for-python echo 0000000000 echo | ls From b08af09282f5a5ce4c28db059c91f7fa03090472 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 17:09:43 +0800 Subject: [PATCH 16/59] check package install --- scripts/auto_release/PythonSdkLiveTest.yml | 7 +----- scripts/auto_release/main.py | 26 ++++++---------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 771843733427..83149fb5e697 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -31,15 +31,12 @@ jobs: # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) mkdir azure-sdk-for-python git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/azure-sdk-for-python - echo 0000000000 - echo | ls # create virtual env python -m venv venv-sdk source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk - echo | pip install packaging - echo | (pip list | grep packaging) + pip install packaging # import env variable export AZURE_TEST_RUN_LIVE=true @@ -53,8 +50,6 @@ jobs: export AZURE_SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) # run - echo 11111111111 - echo | ls cd azure-sdk-for-python python /home/vsts/work/1/s/scripts/auto_release/main.py $(BASE_BRANCH) "$script_path" "$output_path" diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index d63b5e9956dc..524f7661a1fb 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -151,30 +151,16 @@ def edit_file_setup(): def edit_file_readme(): path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' # read and edit usage part - with open(f'{SCRIPT_PATH}/usage.txt', 'r') as file_in: - usage = file_in.readlines() for i in range(0, len(usage)): usage[i] = re.sub('MyService', SERVICE_NAME.capitalize(), usage[i], flags=re.IGNORECASE) # edit README - with open(f'{path}/README.md') as file_in: + with open(f'{path}/README.md', 'r') as file_in: list_in = file_in.readlines() - i = 1 - list_out = [list_in[0]] - while i < len(list_in): - if list_in[i].find('## Usage') > -1: - break - list_out.append(list_in[i]) - i = i + 1 - list_out.extend(usage) - flag = False - while i < len(list_in): - if list_in[i].find('## Provide Feedback') > -1: - flag = True - if flag: - list_out.append(list_in[i]) - i = i + 1 + for i in range(0, len(list_in)): + if list_in[i].find('MyService') > 0: + list_in[i] = list_in[i].replace('MyService', SERVICE_NAME.capitalize()) with open(f'{path}/README.md', 'w') as file_out: - file_out.writelines(list_out) + file_out.writelines(list_in) def edit_first_release(): @@ -225,6 +211,8 @@ def build_wheel(): path = os.getcwd() setup_path = f'{path}/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' print_check(f'cd {setup_path} && python setup.py bdist_wheel') + # check whether package could install + print_check(f'python -c "import azure.mgmt.{SERVICE_NAME}"') print_check(f'cd {path}') From 62fca55c73b65eefd27e7af87f876e9fc6861a73 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 25 Feb 2021 17:20:05 +0800 Subject: [PATCH 17/59] debug --- scripts/auto_release/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 524f7661a1fb..92df3bdc0946 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -150,9 +150,6 @@ def edit_file_setup(): def edit_file_readme(): path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' - # read and edit usage part - for i in range(0, len(usage)): - usage[i] = re.sub('MyService', SERVICE_NAME.capitalize(), usage[i], flags=re.IGNORECASE) # edit README with open(f'{path}/README.md', 'r') as file_in: list_in = file_in.readlines() From 7409ad3d04ec24d2cb945830abc7a998ce163239 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 15:11:05 +0800 Subject: [PATCH 18/59] 'rc' in version && check package && edit version.py in sub folder --- scripts/auto_release/main.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 92df3bdc0946..a2c7c79ed09b 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -103,10 +103,14 @@ def edit_version(add_content): num = VERSION_OLD.split('.') if TRACK == '1' and num[0] == '0': VERSION_NEW = f'0.{str(int(num[1]) + 1)}.0' - elif TRACK == '2' and VERSION_OLD.find('b') > -1: + elif VERSION_OLD.find('b') > -1: lastnum = num[2].split('b') lastnum[1] = str(int(lastnum[1]) + 1) VERSION_NEW = f'{num[0]}.{num[1]}.{lastnum[0]}b{lastnum[1]}' + elif VERSION_OLD.find('rc') > -1: + lastnum = num[2].split('rc') + lastnum[1] = str(int(lastnum[1]) + 1) + VERSION_NEW = f'{num[0]}.{num[1]}.{lastnum[0]}rc{lastnum[1]}' elif flag[0]: VERSION_NEW = f'{int(num[0]) + 1}.0.0' elif flag[1]: @@ -208,12 +212,16 @@ def build_wheel(): path = os.getcwd() setup_path = f'{path}/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' print_check(f'cd {setup_path} && python setup.py bdist_wheel') - # check whether package could install - print_check(f'python -c "import azure.mgmt.{SERVICE_NAME}"') print_check(f'cd {path}') + # check whether package can install + print_check(f'python -c "import azure.mgmt.{SERVICE_NAME}"') + print_check(f'python -m packaging_tools.code_report azure-mgmt-{SERVICE_NAME}') + def test_env_init(): + # need to execute twice + print_exec(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') print_check(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') file = f'{SCRIPT_PATH}/livetest_package_{SERVICE_NAME}_track{TRACK}.txt' if os.path.exists(file): @@ -233,14 +241,18 @@ def run_live_test(): my_print('live test run done !!!') -def del_useless_file(): - if TRACK == '1': - return - +def edit_useless_file(): + file = 'version.py' if TRACK == '1' else '_version.py' path = f'{os.getcwd()}/sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/azure/mgmt/{SERVICE_NAME}' for folder in os.listdir(path): - if os.path.isdir(f'{path}/{folder}') and os.path.exists(f'{path}/{folder}/_version.py'): - print_exec(f'rm -rf {path}/{folder}/_version.py') + if os.path.isdir(f'{path}/{folder}') and os.path.exists(f'{path}/{folder}/{file}'): + with open(f'{path}/{folder}/{file}', 'r') as file_in: + list_in = file_in.readlines() + for i in range(0, len(list_in)): + if list_in[i].find('VERSION') > -1: + list_in[i] = f'VERSION = "{VERSION_NEW}"\n' + with open(f'{path}/{folder}/{file}', 'w') as file_out: + file_out.writelines(list_in) def commit_test(): @@ -318,8 +330,8 @@ def main(): judge_sdk_folder() create_branch() init_env() - del_useless_file() edit_file() + edit_useless_file() check_pprint_name() # commit_file() run_live_test() From b58d2b5f67cb29ab9d7adbcc4e856f9103cda168 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 15:23:01 +0800 Subject: [PATCH 19/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 1 + scripts/auto_release/main.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 83149fb5e697..39718dd09e65 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -37,6 +37,7 @@ jobs: source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk pip install packaging + pip install -r $script_path/livetest_package.txt # import env variable export AZURE_TEST_RUN_LIVE=true diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a2c7c79ed09b..d8c4a4c0d031 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -220,8 +220,7 @@ def build_wheel(): def test_env_init(): - # need to execute twice - print_exec(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') + my_print(print_check(f'pip install -r {SCRIPT_PATH}/livetest_package.txt')) print_check(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') file = f'{SCRIPT_PATH}/livetest_package_{SERVICE_NAME}_track{TRACK}.txt' if os.path.exists(file): From 4464635b65840945ec67d2922a6a1d418e166b1b Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 15:37:16 +0800 Subject: [PATCH 20/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 2 ++ scripts/auto_release/main.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 39718dd09e65..291d1fda19f6 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -37,7 +37,9 @@ jobs: source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk pip install packaging + echo 000000 pip install -r $script_path/livetest_package.txt + echo 111111 # import env variable export AZURE_TEST_RUN_LIVE=true diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index d8c4a4c0d031..3649ccbbed56 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -220,7 +220,7 @@ def build_wheel(): def test_env_init(): - my_print(print_check(f'pip install -r {SCRIPT_PATH}/livetest_package.txt')) + my_print(print_exec_output(f'pip install -r {SCRIPT_PATH}/livetest_package.txt')) print_check(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') file = f'{SCRIPT_PATH}/livetest_package_{SERVICE_NAME}_track{TRACK}.txt' if os.path.exists(file): From 580c69fc8a9da5b138667ed27d7ff120cbc8b533 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 15:52:44 +0800 Subject: [PATCH 21/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 291d1fda19f6..4c1ebf881128 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -24,6 +24,9 @@ jobs: architecture: 'x64' - bash: | script_path=$(pwd) + echo 00000000 + echo | ls + echo 11111111 cd .. git config --global user.email "PythonSdkPipelines" git config --global user.name "PythonSdkPipelines" @@ -37,9 +40,6 @@ jobs: source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk pip install packaging - echo 000000 - pip install -r $script_path/livetest_package.txt - echo 111111 # import env variable export AZURE_TEST_RUN_LIVE=true From 7f215086a3a63be143948423cbb66c7df56e19dc Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 16:46:17 +0800 Subject: [PATCH 22/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 4c1ebf881128..0b324109ab9d 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -23,7 +23,7 @@ jobs: addToPath: true architecture: 'x64' - bash: | - script_path=$(pwd) + script_path=$(pwd)/scripts/auto_release echo 00000000 echo | ls echo 11111111 From 44b39952e74f313d0c10358dcf26411615b51295 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 17:03:19 +0800 Subject: [PATCH 23/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 0b324109ab9d..973f193aa5f7 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -40,6 +40,9 @@ jobs: source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk pip install packaging + echo 2222222222222 + pip install -r $script_path/livetest_package.txt + echo 3333333333333 # import env variable export AZURE_TEST_RUN_LIVE=true @@ -54,7 +57,7 @@ jobs: # run cd azure-sdk-for-python - python /home/vsts/work/1/s/scripts/auto_release/main.py $(BASE_BRANCH) "$script_path" "$output_path" + python $script_path/main.py $(BASE_BRANCH) "$script_path" "$output_path" # judge if [ ! -f "$output_path/output.txt" ]; then From 0f2455b49bf773ebb4ac6d23680cf35a98239830 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 17:13:41 +0800 Subject: [PATCH 24/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 6 ------ scripts/auto_release/main.py | 5 +++-- scripts/auto_release/usage.txt | 10 ---------- 3 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 scripts/auto_release/usage.txt diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 973f193aa5f7..b36bd13063c3 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -24,9 +24,6 @@ jobs: architecture: 'x64' - bash: | script_path=$(pwd)/scripts/auto_release - echo 00000000 - echo | ls - echo 11111111 cd .. git config --global user.email "PythonSdkPipelines" git config --global user.name "PythonSdkPipelines" @@ -40,9 +37,6 @@ jobs: source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk pip install packaging - echo 2222222222222 - pip install -r $script_path/livetest_package.txt - echo 3333333333333 # import env variable export AZURE_TEST_RUN_LIVE=true diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 3649ccbbed56..862baf24fe54 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -220,8 +220,7 @@ def build_wheel(): def test_env_init(): - my_print(print_exec_output(f'pip install -r {SCRIPT_PATH}/livetest_package.txt')) - print_check(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') + print_exec(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') file = f'{SCRIPT_PATH}/livetest_package_{SERVICE_NAME}_track{TRACK}.txt' if os.path.exists(file): print_exec(f'pip install -r {file}') @@ -298,6 +297,8 @@ def judge_sdk_folder(): TRACK = '2' break + my_print(str(FOLDER_LINK)) + def git_remote_add(): global TRACK, NEW_BRANCH diff --git a/scripts/auto_release/usage.txt b/scripts/auto_release/usage.txt deleted file mode 100644 index ed8eafd4d9d5..000000000000 --- a/scripts/auto_release/usage.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Usage - - -To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) - - - -For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/) -Code samples for this package can be found at [MyService Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. -Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) From 0caeed0ab700e1a5fe74301afa279d82964b0d7b Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 17:56:07 +0800 Subject: [PATCH 25/59] debug --- scripts/auto_release/main.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 862baf24fe54..6c9b79b96295 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -225,6 +225,26 @@ def test_env_init(): if os.path.exists(file): print_exec(f'pip install -r {file}') + # edit mgmt_settings_real.py + with open(f'{SCRIPT_PATH}/mgmt_settings_real_.py', 'r') as file_in: + list_in = file_in.readlines() + + ENV_TENANT_ID = os.environ().get('ENV_TENANT_ID') + ENV_CLIENT_ID = os.environ().get('ENV_CLIENT_ID') + ENV_CLIENT_SECRET = os.environ().get('ENV_CLIENT_SECRET') + ENV_SUBSCRIPTION_ID = os.environ().get('ENV_SUBSCRIPTION_ID') + + my_print(f'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx={ENV_TENANT_ID}, {ENV_CLIENT_ID}, {ENV_CLIENT_SECRET}, {ENV_SUBSCRIPTION_ID}') + + for i in range(0, len(list_in)): + list_in[i] = list_in[i].replace('ENV_TENANT_ID', ENV_TENANT_ID) + list_in[i] = list_in[i].replace('ENV_CLIENT_ID', ENV_CLIENT_ID) + list_in[i] = list_in[i].replace('ENV_CLIENT_SECRET', ENV_CLIENT_SECRET) + list_in[i] = list_in[i].replace('ENV_SUBSCRIPTION_ID', ENV_SUBSCRIPTION_ID) + + with open('tools/azure-sdk-tools/devtools_testutils/mgmt_settings_real.py', 'w') as file_out: + file_out.writelines(list_in) + def run_live_test(): test_env_init() @@ -297,8 +317,6 @@ def judge_sdk_folder(): TRACK = '2' break - my_print(str(FOLDER_LINK)) - def git_remote_add(): global TRACK, NEW_BRANCH From de2c863dda2eb51ae9e93a10d3f18f43efe0a536 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 26 Feb 2021 18:00:01 +0800 Subject: [PATCH 26/59] debug --- scripts/auto_release/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 6c9b79b96295..46af856092d8 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -229,10 +229,10 @@ def test_env_init(): with open(f'{SCRIPT_PATH}/mgmt_settings_real_.py', 'r') as file_in: list_in = file_in.readlines() - ENV_TENANT_ID = os.environ().get('ENV_TENANT_ID') - ENV_CLIENT_ID = os.environ().get('ENV_CLIENT_ID') - ENV_CLIENT_SECRET = os.environ().get('ENV_CLIENT_SECRET') - ENV_SUBSCRIPTION_ID = os.environ().get('ENV_SUBSCRIPTION_ID') + ENV_TENANT_ID = os.environ().get('TENANT_ID') + ENV_CLIENT_ID = os.environ().get('CLIENT_ID') + ENV_CLIENT_SECRET = os.environ().get('CLIENT_SECRET') + ENV_SUBSCRIPTION_ID = os.environ().get('SUBSCRIPTION_ID') my_print(f'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx={ENV_TENANT_ID}, {ENV_CLIENT_ID}, {ENV_CLIENT_SECRET}, {ENV_SUBSCRIPTION_ID}') From 97fa9cc8b26507c83e46f6ae3912324874f2ff64 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 1 Mar 2021 10:07:03 +0800 Subject: [PATCH 27/59] debug --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 46af856092d8..4d83697f3b80 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -226,7 +226,7 @@ def test_env_init(): print_exec(f'pip install -r {file}') # edit mgmt_settings_real.py - with open(f'{SCRIPT_PATH}/mgmt_settings_real_.py', 'r') as file_in: + with open(f'{SCRIPT_PATH}/mgmt_settings_real.py', 'r') as file_in: list_in = file_in.readlines() ENV_TENANT_ID = os.environ().get('TENANT_ID') From 3a5ca981308b9797bbb6640d6a6590cde488b974 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 1 Mar 2021 10:27:21 +0800 Subject: [PATCH 28/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index b36bd13063c3..e48cbf7da9d0 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -24,7 +24,16 @@ jobs: architecture: 'x64' - bash: | script_path=$(pwd)/scripts/auto_release + echo 00000 + cd scripts + echo | ls + echo 1111 + cd auto_release + echo | ls cd .. + cd .. + cd .. + echo 2222 git config --global user.email "PythonSdkPipelines" git config --global user.name "PythonSdkPipelines" @@ -37,6 +46,9 @@ jobs: source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk pip install packaging + echoo 333333333 + pip install -r $script_path/livetest_package.txt + echoo 444444444 # import env variable export AZURE_TEST_RUN_LIVE=true From a72a157ead1043d92dc86f1abba10ddefce3d077 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 1 Mar 2021 10:41:59 +0800 Subject: [PATCH 29/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 8 ++- scripts/auto_release/main.py | 2 +- scripts/auto_release/mgmt_settings_real_.py | 73 +++++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 scripts/auto_release/mgmt_settings_real_.py diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index e48cbf7da9d0..9b5ab43c0221 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -46,9 +46,9 @@ jobs: source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk pip install packaging - echoo 333333333 + echo 333333333 pip install -r $script_path/livetest_package.txt - echoo 444444444 + echo 444444444 # import env variable export AZURE_TEST_RUN_LIVE=true @@ -61,6 +61,10 @@ jobs: export AZURE_CLIENT_SECRET=$(ENV_CLIENT_SECRET) export AZURE_SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) + echo 5555555 + echo TENANT_ID $TENANT_ID + echo 6666666 + # run cd azure-sdk-for-python python $script_path/main.py $(BASE_BRANCH) "$script_path" "$output_path" diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 4d83697f3b80..46af856092d8 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -226,7 +226,7 @@ def test_env_init(): print_exec(f'pip install -r {file}') # edit mgmt_settings_real.py - with open(f'{SCRIPT_PATH}/mgmt_settings_real.py', 'r') as file_in: + with open(f'{SCRIPT_PATH}/mgmt_settings_real_.py', 'r') as file_in: list_in = file_in.readlines() ENV_TENANT_ID = os.environ().get('TENANT_ID') diff --git a/scripts/auto_release/mgmt_settings_real_.py b/scripts/auto_release/mgmt_settings_real_.py new file mode 100644 index 000000000000..f34ba89fe11e --- /dev/null +++ b/scripts/auto_release/mgmt_settings_real_.py @@ -0,0 +1,73 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +from azure.common.credentials import ( + BasicTokenAuthentication, + UserPassCredentials, + ServicePrincipalCredentials, +) + +SUBSCRIPTION_ID = "ENV_SUBSCRIPTION_ID" +CLIENT_ID = 'ENV_CLIENT_ID' +CLIENT_SECRET = 'ENV_CLIENT_SECRET' + +# Keyvault tests +TENANT_ID = 'ENV_TENANT_ID' +# Cognitive Services tests +CS_SUBSCRIPTION_KEY = '0000000000000000000000000000' +# Event Grid key +EVENT_GRID_KEY = '0000000000000000000000000000' +# HDInsight tests +HDI_ADLS_ACCOUNT_NAME = 'fakehdiadlsaccount' +HDI_ADLS_CLIENT_ID = '00000000-0000-0000-0000-000000000000' + +# Ubuntu image +LINUX_OS_VHD = "https://mystorageaccount.blob.core.windows.net/inputtestdatadonotdelete/ubuntu.vhd" + +# Storage tests related +ACTIVE_DIRECTORY_APPLICATION_ID = '00000000-0000-0000-0000-00000000000' +ACTIVE_DIRECTORY_APPLICATION_SECRET = '000000000ft5g5g5g5g5g5g5g5000000?' +ACTIVE_DIRECTORY_TENANT_ID = '00000000-0000-0000-0000-000000000000' +IS_SERVER_SIDE_FILE_ENCRYPTION_ENABLED = True +ENABLE_LOGGING = True +AZURE_STORAGE_CONNECTION_STRING = 'DefaultEndpointsProtocol=https;AccountName=storagename;AccountKey=NzhL3hKZbJBuJ2484dPTR+xF30kYaWSSCbs2BzLgVVI1woqeST/1IgqaLm6QAOTxtGvxctSNbIR/1hW8yH+bJg==;EndpointSuffix=core.windows.net' + +# Read for details of this file: +# https://github.com/Azure/azure-sdk-for-python/wiki/Contributing-to-the-tests + +def get_azure_core_credentials(**kwargs): + from azure.identity import ClientSecretCredential + return ClientSecretCredential( + client_id = 'ENV_CLIENT_ID', + client_secret = 'ENV_CLIENT_SECRET', + tenant_id = 'ENV_TENANT_ID' + ) + # # Needed to play recorded tests + # from azure.core.credentials import AccessToken + # class FakeCredential(object): + # def get_token(self, *scopes, **kwargs): + # return AccessToken('fake_token', 2527537086) + # return FakeCredential() + +def get_credentials(**kwargs): + # Put your credentials here in the "real" file + #return UserPassCredentials( + # 'user@myaddomain.onmicrosoft.com', + # 'Password' + #) + # note that UserCredential does not work any longer. Must use a ServicePrincipal. + # for deprecated APIs I believe will still work. + return ServicePrincipalCredentials( + client_id = 'ENV_CLIENT_ID', + secret = 'ENV_CLIENT_SECRET', + tenant = 'ENV_TENANT_ID' + ) + # Needed to play recorded tests + # return BasicTokenAuthentication( + # token = { + # 'access_token':'faked_token' + # } + # ) \ No newline at end of file From 1b1d24ff6395fc03fd34beae01490332b1554f21 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 1 Mar 2021 10:58:22 +0800 Subject: [PATCH 30/59] debug --- scripts/auto_release/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 46af856092d8..39aca16bb1ef 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -229,10 +229,10 @@ def test_env_init(): with open(f'{SCRIPT_PATH}/mgmt_settings_real_.py', 'r') as file_in: list_in = file_in.readlines() - ENV_TENANT_ID = os.environ().get('TENANT_ID') - ENV_CLIENT_ID = os.environ().get('CLIENT_ID') - ENV_CLIENT_SECRET = os.environ().get('CLIENT_SECRET') - ENV_SUBSCRIPTION_ID = os.environ().get('SUBSCRIPTION_ID') + ENV_TENANT_ID = os.environ['TENANT_ID'] + ENV_CLIENT_ID = os.environ['CLIENT_ID'] + ENV_CLIENT_SECRET = os.environ['CLIENT_SECRET'] + ENV_SUBSCRIPTION_ID = os.environ['SUBSCRIPTION_ID'] my_print(f'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx={ENV_TENANT_ID}, {ENV_CLIENT_ID}, {ENV_CLIENT_SECRET}, {ENV_SUBSCRIPTION_ID}') From d9a380a82a42344d2eb6d2ee451854a1aaa789f0 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 1 Mar 2021 14:07:00 +0800 Subject: [PATCH 31/59] debug --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 39aca16bb1ef..e983c19b6e12 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -251,7 +251,7 @@ def run_live_test(): print_exec(f'python scripts/dev_setup.py -p azure-mgmt-{SERVICE_NAME}') # run live test try: - print_check(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') + result = print_exec_output(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') except: my_print('some test failed, please fix it locally') print_check(f'touch {OUT_PATH}/live_test_fail.txt') From f963a37448a78134207554a7754c1f8d478c69fb Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 1 Mar 2021 16:23:14 +0800 Subject: [PATCH 32/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 13 +++++++++---- scripts/auto_release/main.py | 19 ++++++++++--------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 9b5ab43c0221..60b306416fcd 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -75,6 +75,11 @@ jobs: exit 1 fi + test_result = '' + if [ -f "$output_path/live_test_fail.txt" ]; then + test_result = `cat $output_path/live_test_fail.txt` + fi + # create PR new_branch = `sed -n '1p' $output_path/output.txt` target_branch = `sed -n '2p' $output_path/output.txt` @@ -82,7 +87,7 @@ jobs: --user "anything:$(USR_TOKEN)" \ -d "{\ \"title\":\"[AutoRelease_Python] $new_branch\",\ - \"body\":\"Please Add the link issue\",\ + \"body\":\"Please Add the link issue\n$test_result\",\ \"head\":\"$(USR_NAME):$new_branch\",\ \"base\":\"$target_branch\"\ }" \ @@ -90,8 +95,8 @@ jobs: echo "\'[AutoRelease_Python] $new_branch \' has been created!!!" - # if test failed, still push and crete PR. But pipeline failed to remind user + # if test failed, still push and crete PR. But pipeline would fail to remind user if [ -f "$output_path/live_test_fail.txt" ]; then - echo "please fix failed test!!! \'[AutoRelease_Python] $new_branch \' has been created!!! " + echo "please fix failed test!!! \'[AutoRelease_Python] $new_branch \' has been created!!!" exit 1 - fi + fi \ No newline at end of file diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index e983c19b6e12..713ff54365bf 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -250,13 +250,14 @@ def run_live_test(): test_env_init() print_exec(f'python scripts/dev_setup.py -p azure-mgmt-{SERVICE_NAME}') # run live test - try: - result = print_exec_output(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') - except: - my_print('some test failed, please fix it locally') - print_check(f'touch {OUT_PATH}/live_test_fail.txt') - else: - my_print('live test run done !!!') + result = print_exec_output(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') + for line in result: + if line.find('FAILED ') > -1: + with open(f'{OUT_PATH}/live_test_fail.txt', 'w') as file_out: + file_out.writelines(result) + my_print('some test failed, please fix it locally') + return + my_print('live test run done, do not find failure !!!') def edit_useless_file(): @@ -351,10 +352,10 @@ def main(): edit_file() edit_useless_file() check_pprint_name() - # commit_file() + commit_file() run_live_test() build_wheel() - # commit_test() + commit_test() if __name__ == '__main__': From 5b4f9f9a1433c68349e86f5047e0ae2e2b024aa0 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 1 Mar 2021 16:57:28 +0800 Subject: [PATCH 33/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 60b306416fcd..6fc24adfcda3 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -77,12 +77,16 @@ jobs: test_result = '' if [ -f "$output_path/live_test_fail.txt" ]; then - test_result = `cat $output_path/live_test_fail.txt` + test_result=`cat $output_path/live_test_fail.txt` fi # create PR - new_branch = `sed -n '1p' $output_path/output.txt` - target_branch = `sed -n '2p' $output_path/output.txt` + new_branch=`sed -n '1p' $output_path/output.txt` + target_branch=`sed -n '2p' $output_path/output.txt` + echo 777777777 + echo new_branch: $new_branch + echo target_branch: $target_branch + echo 88888888 curl \ --user "anything:$(USR_TOKEN)" \ -d "{\ From 85d3a8a9f5104840e95a4edcf1eb730283959702 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 1 Mar 2021 17:19:13 +0800 Subject: [PATCH 34/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 4 +++- scripts/auto_release/main.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 6fc24adfcda3..53228672d4f3 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -78,6 +78,7 @@ jobs: test_result = '' if [ -f "$output_path/live_test_fail.txt" ]; then test_result=`cat $output_path/live_test_fail.txt` + echo test_result: $test_result fi # create PR @@ -86,12 +87,13 @@ jobs: echo 777777777 echo new_branch: $new_branch echo target_branch: $target_branch + echo test_result: $test_result echo 88888888 curl \ --user "anything:$(USR_TOKEN)" \ -d "{\ \"title\":\"[AutoRelease_Python] $new_branch\",\ - \"body\":\"Please Add the link issue\n$test_result\",\ + \"body\":\"Please Add the link issue\n \",\ \"head\":\"$(USR_NAME):$new_branch\",\ \"base\":\"$target_branch\"\ }" \ diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 713ff54365bf..468e3c051317 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -333,7 +333,7 @@ def create_branch(): # create new branch t = time.time() d = time.localtime(t) - NEW_BRANCH = 't{}-{}-{}-{:02d}-{:02d}-{}'.format(TRACK, SERVICE_NAME, d.tm_year, d.tm_mon, d.tm_mday, str(t)[-6:]) + NEW_BRANCH = 't{}-{}-{}-{:02d}-{:02d}-{}'.format(TRACK, SERVICE_NAME, d.tm_year, d.tm_mon, d.tm_mday, str(t)[-5:]) print_exec(f'git checkout -b {NEW_BRANCH}') From 057ed90b7eb3fda9a14f270abe1120bd569a4cc6 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 13:40:03 +0800 Subject: [PATCH 35/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 53228672d4f3..9cb8cd92b8a8 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -93,11 +93,11 @@ jobs: --user "anything:$(USR_TOKEN)" \ -d "{\ \"title\":\"[AutoRelease_Python] $new_branch\",\ - \"body\":\"Please Add the link issue\n \",\ + \"body\":\"Please Add the link issue\n $test_result \",\ \"head\":\"$(USR_NAME):$new_branch\",\ \"base\":\"$target_branch\"\ }" \ - $(REPO)/pulls + https://api.github.com/repos/Azure/azure-sdk-for-python/pulls echo "\'[AutoRelease_Python] $new_branch \' has been created!!!" From 3dbe067785d2661d918124726f2bda690bb18d4d Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 14:01:38 +0800 Subject: [PATCH 36/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 22 +++------------------- scripts/auto_release/main.py | 2 -- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 9cb8cd92b8a8..1110be7d63af 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -24,16 +24,7 @@ jobs: architecture: 'x64' - bash: | script_path=$(pwd)/scripts/auto_release - echo 00000 - cd scripts - echo | ls - echo 1111 - cd auto_release - echo | ls cd .. - cd .. - cd .. - echo 2222 git config --global user.email "PythonSdkPipelines" git config --global user.name "PythonSdkPipelines" @@ -46,9 +37,7 @@ jobs: source venv-sdk/bin/activate output_path=$(pwd)/venv-sdk pip install packaging - echo 333333333 pip install -r $script_path/livetest_package.txt - echo 444444444 # import env variable export AZURE_TEST_RUN_LIVE=true @@ -61,10 +50,6 @@ jobs: export AZURE_CLIENT_SECRET=$(ENV_CLIENT_SECRET) export AZURE_SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) - echo 5555555 - echo TENANT_ID $TENANT_ID - echo 6666666 - # run cd azure-sdk-for-python python $script_path/main.py $(BASE_BRANCH) "$script_path" "$output_path" @@ -75,10 +60,9 @@ jobs: exit 1 fi - test_result = '' + test_result="" if [ -f "$output_path/live_test_fail.txt" ]; then - test_result=`cat $output_path/live_test_fail.txt` - echo test_result: $test_result + test_result="Live test failed, detailed info is in pipeline log!!!\n" fi # create PR @@ -93,7 +77,7 @@ jobs: --user "anything:$(USR_TOKEN)" \ -d "{\ \"title\":\"[AutoRelease_Python] $new_branch\",\ - \"body\":\"Please Add the link issue\n $test_result \",\ + \"body\":\"$test_result Please Add the link issue \",\ \"head\":\"$(USR_NAME):$new_branch\",\ \"base\":\"$target_branch\"\ }" \ diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 468e3c051317..93105436e7f1 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -234,8 +234,6 @@ def test_env_init(): ENV_CLIENT_SECRET = os.environ['CLIENT_SECRET'] ENV_SUBSCRIPTION_ID = os.environ['SUBSCRIPTION_ID'] - my_print(f'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx={ENV_TENANT_ID}, {ENV_CLIENT_ID}, {ENV_CLIENT_SECRET}, {ENV_SUBSCRIPTION_ID}') - for i in range(0, len(list_in)): list_in[i] = list_in[i].replace('ENV_TENANT_ID', ENV_TENANT_ID) list_in[i] = list_in[i].replace('ENV_CLIENT_ID', ENV_CLIENT_ID) From e9ef6c57f4714d01e4461465e9a88b811dc4e505 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 14:49:31 +0800 Subject: [PATCH 37/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 1110be7d63af..648dd3fb3d43 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -68,25 +68,20 @@ jobs: # create PR new_branch=`sed -n '1p' $output_path/output.txt` target_branch=`sed -n '2p' $output_path/output.txt` - echo 777777777 - echo new_branch: $new_branch - echo target_branch: $target_branch - echo test_result: $test_result - echo 88888888 curl \ --user "anything:$(USR_TOKEN)" \ -d "{\ - \"title\":\"[AutoRelease_Python] $new_branch\",\ + \"title\":\"[AutoRelease] $new_branch\",\ \"body\":\"$test_result Please Add the link issue \",\ \"head\":\"$(USR_NAME):$new_branch\",\ \"base\":\"$target_branch\"\ }" \ https://api.github.com/repos/Azure/azure-sdk-for-python/pulls - echo "\'[AutoRelease_Python] $new_branch \' has been created!!!" + echo "\'[AutoRelease] $new_branch \' has been created!!!" # if test failed, still push and crete PR. But pipeline would fail to remind user if [ -f "$output_path/live_test_fail.txt" ]; then - echo "please fix failed test!!! \'[AutoRelease_Python] $new_branch \' has been created!!!" + echo "please fix failed test!!! \'[AutoRelease] $new_branch \' has been created!!!" exit 1 fi \ No newline at end of file From a7361d6dc4109840ddaa6751125f93d0b959c8a6 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 16:00:14 +0800 Subject: [PATCH 38/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- scripts/auto_release/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 648dd3fb3d43..ddd102100e11 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -71,7 +71,7 @@ jobs: curl \ --user "anything:$(USR_TOKEN)" \ -d "{\ - \"title\":\"[AutoRelease] $new_branch\",\ + \"title\":\"[AutoRelease] $new_branch(Do not merge)\",\ \"body\":\"$test_result Please Add the link issue \",\ \"head\":\"$(USR_NAME):$new_branch\",\ \"base\":\"$target_branch\"\ diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 93105436e7f1..a0693d6c5726 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -180,7 +180,7 @@ def edit_first_release(): data_format = '{}-{:02d}-{:02d}'.format(date.tm_year, date.tm_mon, date.tm_mday) for i in range(0, len(content)): content[i] = content[i].replace('data_format', data_format) - with open(f'{path}/CHANGELOG.md', 'w') as file_out: + with open(f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/CHANGELOG.md', 'w') as file_out: file_out.writelines(content) From 4cbd763346cce4a32f1cd29dfc1d446ce2336ff7 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 16:15:41 +0800 Subject: [PATCH 39/59] debug --- scripts/auto_release/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a0693d6c5726..5d812781d271 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -146,8 +146,10 @@ def edit_file_setup(): path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' with open(f'{path}/setup.py', 'r') as file_in: list_in = file_in.readlines() + my_print('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') for i in range(0, len(list_in)): list_in[i] = list_in[i].replace('msrestazure>=0.4.32,<2.0.0', 'azure-mgmt-core>=1.2.0,<2.0.0') + my_print(list_in[i]) with open(f'{path}/setup.py', 'w') as file_out: file_out.writelines(list_in) From cf396c8de9153cf0baf894a4c1672901181f9c19 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 16:36:47 +0800 Subject: [PATCH 40/59] debug --- scripts/auto_release/main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 5d812781d271..58efcfedaa00 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -146,10 +146,8 @@ def edit_file_setup(): path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' with open(f'{path}/setup.py', 'r') as file_in: list_in = file_in.readlines() - my_print('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') for i in range(0, len(list_in)): list_in[i] = list_in[i].replace('msrestazure>=0.4.32,<2.0.0', 'azure-mgmt-core>=1.2.0,<2.0.0') - my_print(list_in[i]) with open(f'{path}/setup.py', 'w') as file_out: file_out.writelines(list_in) @@ -194,6 +192,9 @@ def edit_file(): except: print_changelog(['* Initial Release']) edit_first_release() + edit_file_readme() + if TRACK == '2': + edit_file_setup() my_print(f'CHANGELOG and version(new:{VERSION_NEW}) generate successfully. It is first release') else: add_content = create_changelog_content() @@ -203,9 +204,9 @@ def edit_file(): print_changelog(add_content) edit_version(add_content) edit_changelog(add_content) + edit_file_readme() if TRACK == '2': edit_file_setup() - edit_file_readme() my_print(f'CHANGELOG and version(new:{VERSION_NEW}) generate successfully, please check it(compare with ' f'{VERSION_LAST_RELEASE}[https://pypi.org/pypi/azure-mgmt-{SERVICE_NAME}/{VERSION_LAST_RELEASE}])') @@ -297,7 +298,7 @@ def check_pprint_name(): list_in[i] = list_in[i].replace('MyService', pprint_name) with open(file_path, 'w') as file_out: file_out.writelines(list_in) - my_print(f'== replace\"MyService\" with \"{pprint_name}\" successfully ==') + my_print(f' replace \"MyService\" with \"{pprint_name}\" successfully ') def judge_sdk_folder(): @@ -386,3 +387,4 @@ def main(): else: with open(f'{OUT_PATH}/output.txt', 'w') as file_out: file_out.writelines([f'{NEW_BRANCH}\n', "master" if TRACK == '2' else 'release/v3']) +` \ No newline at end of file From ac11d5847fd8c6cff57b86e4fbbd6bfe9bc35990 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 16:54:57 +0800 Subject: [PATCH 41/59] debug --- scripts/auto_release/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 58efcfedaa00..c46cbdafd08f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -387,4 +387,3 @@ def main(): else: with open(f'{OUT_PATH}/output.txt', 'w') as file_out: file_out.writelines([f'{NEW_BRANCH}\n', "master" if TRACK == '2' else 'release/v3']) -` \ No newline at end of file From ebaf8ceda3e9c3fd604073072642c8d72c15ce00 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 17:21:37 +0800 Subject: [PATCH 42/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- scripts/auto_release/main.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index ddd102100e11..059399f2ca64 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -62,7 +62,7 @@ jobs: test_result="" if [ -f "$output_path/live_test_fail.txt" ]; then - test_result="Live test failed, detailed info is in pipeline log!!!\n" + test_result="Live test failed, detailed info is in pipeline log(search keyword \"FAILED \")!!!\n" fi # create PR diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index c46cbdafd08f..15374dc8bc8c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -251,14 +251,14 @@ def run_live_test(): test_env_init() print_exec(f'python scripts/dev_setup.py -p azure-mgmt-{SERVICE_NAME}') # run live test - result = print_exec_output(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') - for line in result: - if line.find('FAILED ') > -1: - with open(f'{OUT_PATH}/live_test_fail.txt', 'w') as file_out: - file_out.writelines(result) - my_print('some test failed, please fix it locally') - return - my_print('live test run done, do not find failure !!!') + try: + print_check(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') + except: + with open(f'{OUT_PATH}/live_test_fail.txt', 'w') as file_out: + file_out.writelines(['']) + my_print('some test failed, please fix it locally') + else: + my_print('live test run done, do not find failure !!!') def edit_useless_file(): From d419e7cd392db0e8737b1b0f2dca2bdf74d67982 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 2 Mar 2021 17:30:05 +0800 Subject: [PATCH 43/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 059399f2ca64..137463f26806 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -62,7 +62,7 @@ jobs: test_result="" if [ -f "$output_path/live_test_fail.txt" ]; then - test_result="Live test failed, detailed info is in pipeline log(search keyword \"FAILED \")!!!\n" + test_result="Live test failed, detailed info is in pipeline log(search keyword FAILED)!!!\n" fi # create PR From fc6e3bef4d3d6a992081d3b03f0cf2a0515797f9 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 4 Mar 2021 14:32:31 +0800 Subject: [PATCH 44/59] debug --- scripts/auto_release/main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 15374dc8bc8c..c8ea5fbd55f6 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -4,6 +4,8 @@ import time import re import argparse +import logging + SERVICE_NAME = 'servicename' SDK_FOLDER = 'servicename' @@ -15,9 +17,10 @@ OUT_PATH = '' NEW_BRANCH = '' +_LOG = logging.getLogger() def my_print(cmd): - print(f'({SERVICE_NAME})==' + cmd + ' ==\n') + _LOG.info(f'({SERVICE_NAME})==' + cmd + ' ==\n') def print_exec(cmd): @@ -139,7 +142,7 @@ def edit_changelog(add_content): def print_changelog(add_content): for line in add_content: - print('[CHANGELOG] ' + line) + _LOG.info('[CHANGELOG] ' + line) def edit_file_setup(): @@ -369,6 +372,10 @@ def main(): parser.add_argument("out_path", help="path where the output is") args = parser.parse_args() + main_logger = logging.getLogger() + logging.basicConfig() + main_logger.setLevel(logging.INFO) + BRANCH_BASE = args.branch.replace('AzureSDKAutomation:', '') SCRIPT_PATH = args.script_path OUT_PATH = args.out_path From 303099fb36b66d63c4ed4b4fd4f31e96f7fd72d1 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 4 Mar 2021 14:44:56 +0800 Subject: [PATCH 45/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 137463f26806..74660d2a4498 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -62,7 +62,7 @@ jobs: test_result="" if [ -f "$output_path/live_test_fail.txt" ]; then - test_result="Live test failed, detailed info is in pipeline log(search keyword FAILED)!!!\n" + test_result="Live test fail, detailed info is in pipeline log(search keyword FAILED)!!!\n" fi # create PR @@ -80,8 +80,8 @@ jobs: echo "\'[AutoRelease] $new_branch \' has been created!!!" - # if test failed, still push and crete PR. But pipeline would fail to remind user + # if test fail, still push and crete PR. But pipeline would fail to remind user if [ -f "$output_path/live_test_fail.txt" ]; then - echo "please fix failed test!!! \'[AutoRelease] $new_branch \' has been created!!!" + echo "please fix failure test!!! \'[AutoRelease] $new_branch \' has been created!!!" exit 1 fi \ No newline at end of file From e3c3055a23efccb997f6ad624901b0dc16e2ad78 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 5 Mar 2021 10:36:14 +0800 Subject: [PATCH 46/59] livetest package --- scripts/auto_release/livetest_package_hdinsight_track1.txt | 4 ++++ scripts/auto_release/livetest_package_hdinsight_track2.txt | 4 ++++ scripts/auto_release/livetest_package_monitor_track1.txt | 6 ++++++ scripts/auto_release/livetest_package_monitor_track2.txt | 6 ++++++ scripts/auto_release/livetest_package_network_track2.txt | 3 +++ 5 files changed, 23 insertions(+) create mode 100644 scripts/auto_release/livetest_package_hdinsight_track1.txt create mode 100644 scripts/auto_release/livetest_package_hdinsight_track2.txt create mode 100644 scripts/auto_release/livetest_package_monitor_track1.txt create mode 100644 scripts/auto_release/livetest_package_monitor_track2.txt create mode 100644 scripts/auto_release/livetest_package_network_track2.txt diff --git a/scripts/auto_release/livetest_package_hdinsight_track1.txt b/scripts/auto_release/livetest_package_hdinsight_track1.txt new file mode 100644 index 000000000000..e1d2f9428210 --- /dev/null +++ b/scripts/auto_release/livetest_package_hdinsight_track1.txt @@ -0,0 +1,4 @@ +azure-keyvault==4.1.0 +azure-keyvault-certificates==4.2.1 +azure-keyvault-keys==4.2.0 +azure-keyvault-secrets==4.2.0 \ No newline at end of file diff --git a/scripts/auto_release/livetest_package_hdinsight_track2.txt b/scripts/auto_release/livetest_package_hdinsight_track2.txt new file mode 100644 index 000000000000..e1d2f9428210 --- /dev/null +++ b/scripts/auto_release/livetest_package_hdinsight_track2.txt @@ -0,0 +1,4 @@ +azure-keyvault==4.1.0 +azure-keyvault-certificates==4.2.1 +azure-keyvault-keys==4.2.0 +azure-keyvault-secrets==4.2.0 \ No newline at end of file diff --git a/scripts/auto_release/livetest_package_monitor_track1.txt b/scripts/auto_release/livetest_package_monitor_track1.txt new file mode 100644 index 000000000000..d4a94581741c --- /dev/null +++ b/scripts/auto_release/livetest_package_monitor_track1.txt @@ -0,0 +1,6 @@ +azure-mgmt-storage +azure-mgmt-eventhub +azure-mgmt-loganalytics +azure-mgmt-compute +azure-mgmt-network +azure-mgmt-logic \ No newline at end of file diff --git a/scripts/auto_release/livetest_package_monitor_track2.txt b/scripts/auto_release/livetest_package_monitor_track2.txt new file mode 100644 index 000000000000..d4a94581741c --- /dev/null +++ b/scripts/auto_release/livetest_package_monitor_track2.txt @@ -0,0 +1,6 @@ +azure-mgmt-storage +azure-mgmt-eventhub +azure-mgmt-loganalytics +azure-mgmt-compute +azure-mgmt-network +azure-mgmt-logic \ No newline at end of file diff --git a/scripts/auto_release/livetest_package_network_track2.txt b/scripts/auto_release/livetest_package_network_track2.txt new file mode 100644 index 000000000000..c6806d9167ce --- /dev/null +++ b/scripts/auto_release/livetest_package_network_track2.txt @@ -0,0 +1,3 @@ +azure-mgmt-compute==14.0.0 +azure-mgmt-storage==11.2.0 +azure-mgmt-privatedns==0.1.0 From a91ad8210d2ac8fd8d2f772f337cd4202dbdbb67 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 5 Mar 2021 14:23:24 +0800 Subject: [PATCH 47/59] live test package --- scripts/auto_release/livetest_package_monitor_track2.txt | 2 +- scripts/auto_release/livetest_package_resource_track2.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 scripts/auto_release/livetest_package_resource_track2.txt diff --git a/scripts/auto_release/livetest_package_monitor_track2.txt b/scripts/auto_release/livetest_package_monitor_track2.txt index d4a94581741c..d666f19e3f87 100644 --- a/scripts/auto_release/livetest_package_monitor_track2.txt +++ b/scripts/auto_release/livetest_package_monitor_track2.txt @@ -1,6 +1,6 @@ azure-mgmt-storage azure-mgmt-eventhub azure-mgmt-loganalytics -azure-mgmt-compute +azure-mgmt-compute==14.0.0 azure-mgmt-network azure-mgmt-logic \ No newline at end of file diff --git a/scripts/auto_release/livetest_package_resource_track2.txt b/scripts/auto_release/livetest_package_resource_track2.txt new file mode 100644 index 000000000000..7d1b0e76dce9 --- /dev/null +++ b/scripts/auto_release/livetest_package_resource_track2.txt @@ -0,0 +1,2 @@ +azure-mgmt-managementgroups +azure-mgmt-msi \ No newline at end of file From 5862a6e1395a118af378450b1b61d95b1487449f Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 5 Mar 2021 14:41:24 +0800 Subject: [PATCH 48/59] debug --- scripts/auto_release/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index c8ea5fbd55f6..c206965e06ed 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -228,6 +228,7 @@ def build_wheel(): def test_env_init(): print_exec(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') file = f'{SCRIPT_PATH}/livetest_package_{SERVICE_NAME}_track{TRACK}.txt' + my_print(file) if os.path.exists(file): print_exec(f'pip install -r {file}') From 6d8fbfa9c545bb10c024c3422124b6953f1d1d1f Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 5 Mar 2021 15:07:02 +0800 Subject: [PATCH 49/59] debug --- scripts/auto_release/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index c206965e06ed..3670df9f3e4c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -228,9 +228,10 @@ def build_wheel(): def test_env_init(): print_exec(f'pip install -r {SCRIPT_PATH}/livetest_package.txt') file = f'{SCRIPT_PATH}/livetest_package_{SERVICE_NAME}_track{TRACK}.txt' - my_print(file) if os.path.exists(file): print_exec(f'pip install -r {file}') + else: + my_print(f'{file} does not exist') # edit mgmt_settings_real.py with open(f'{SCRIPT_PATH}/mgmt_settings_real_.py', 'r') as file_in: From ded817809b01e889f374fa00520b7025dba9612e Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 5 Mar 2021 15:16:23 +0800 Subject: [PATCH 50/59] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 74660d2a4498..47b219fe7412 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -1,6 +1,6 @@ # Live Test For Python SDK -name: PYTHON SDK AUTO RELEASE RUN $(BASE_BRANCH) $(Date:yyyyMMdd)$(Rev:.r) +name: PYTHON SDK AUTO RELEASE trigger: branches: From cb678562b56d3ee02e03841b23d13c5d059cf3ad Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 5 Mar 2021 17:00:18 +0800 Subject: [PATCH 51/59] debug --- scripts/auto_release/livetest_package_compute_track2.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 scripts/auto_release/livetest_package_compute_track2.txt diff --git a/scripts/auto_release/livetest_package_compute_track2.txt b/scripts/auto_release/livetest_package_compute_track2.txt new file mode 100644 index 000000000000..8781009d5481 --- /dev/null +++ b/scripts/auto_release/livetest_package_compute_track2.txt @@ -0,0 +1 @@ +azure-storage \ No newline at end of file From 4fee90e1ca00b66854e6535b3d4e956a4d9d17a9 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 5 Mar 2021 17:02:54 +0800 Subject: [PATCH 52/59] title --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 47b219fe7412..3f304b59306f 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -1,6 +1,6 @@ # Live Test For Python SDK -name: PYTHON SDK AUTO RELEASE +name: SDK AUTO RELEASE $(BASE_BRANCH) trigger: branches: From fcfc24e1d125181e3305338f4bbab3f044bf6981 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 8 Mar 2021 10:23:38 +0800 Subject: [PATCH 53/59] livetest package for compute --- scripts/auto_release/livetest_package_compute_track2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/livetest_package_compute_track2.txt b/scripts/auto_release/livetest_package_compute_track2.txt index 8781009d5481..32e424b6b027 100644 --- a/scripts/auto_release/livetest_package_compute_track2.txt +++ b/scripts/auto_release/livetest_package_compute_track2.txt @@ -1 +1 @@ -azure-storage \ No newline at end of file +azure-storage-blob \ No newline at end of file From a6b8867bf7d094682b0bbbf184eeb2a4add50039 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 8 Mar 2021 10:30:47 +0800 Subject: [PATCH 54/59] title --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 3f304b59306f..3d609d16d22e 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -1,6 +1,6 @@ # Live Test For Python SDK -name: SDK AUTO RELEASE $(BASE_BRANCH) +name: $(BASE_BRANCH) trigger: branches: From 2a469f719845f3baf0a23b987078a7e9f6f51971 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 22 Mar 2021 13:28:40 +0800 Subject: [PATCH 55/59] avoid fake error --- scripts/auto_release/main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 3670df9f3e4c..a3ee693bd047 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -256,6 +256,12 @@ def run_live_test(): test_env_init() print_exec(f'python scripts/dev_setup.py -p azure-mgmt-{SERVICE_NAME}') # run live test + try: + print_check(f'pytest sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/ --collect-only') + except: + my_print('live test run done, do not find any test !!!') + return + try: print_check(f'pytest -s sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}/') except: From 7cd993862200e5dd09a12bd3c6c8324a70f1d857 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Wed, 24 Mar 2021 16:30:39 +0800 Subject: [PATCH 56/59] get last version from pypi --- scripts/auto_release/main.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a3ee693bd047..8af0b4a63411 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -2,7 +2,6 @@ import sys import subprocess as sp import time -import re import argparse import logging @@ -11,14 +10,14 @@ SDK_FOLDER = 'servicename' TRACK = '1' VERSION_NEW = '0.0.0' -VERSION_OLD = '0.0.0' -VERSION_LAST_RELEASE = '0.0.0' +VERSION_LAST_RELEASE = '1.0.0b1' BRANCH_BASE = '' OUT_PATH = '' NEW_BRANCH = '' _LOG = logging.getLogger() + def my_print(cmd): _LOG.info(f'({SERVICE_NAME})==' + cmd + ' ==\n') @@ -60,7 +59,8 @@ def get_version(report): pattern = 'code_reports/' idx1 = report.find(pattern) idx2 = report.find('/', idx1 + len(pattern)) - VERSION_LAST_RELEASE = report[idx1 + len(pattern):idx2] + if idx2 > -1 and idx1 > -1: + VERSION_LAST_RELEASE = report[idx1 + len(pattern):idx2] def create_changelog_content(): @@ -79,7 +79,7 @@ def create_changelog_content(): def edit_version(add_content): - global VERSION_NEW, VERSION_OLD + global VERSION_NEW, VERSION_LAST_RELEASE flag = [False, False, False] # breaking, feature, bugfix for line in add_content: if line.find('**Breaking changes**') > -1: @@ -94,23 +94,15 @@ def edit_version(add_content): file_name = 'version.py' if TRACK == '1' else '_version.py' with open(f'{path}/{file_name}', 'r') as file_in: list_in = file_in.readlines() - i = 0 - while i < len(list_in): - result = re.search('\".+\"', list_in[i]) - if result: - idx = result.span() - VERSION_OLD = list_in[i][idx[0] + 1: idx[1] - 1] - break - i = i + 1 - num = VERSION_OLD.split('.') + num = VERSION_LAST_RELEASE.split('.') if TRACK == '1' and num[0] == '0': VERSION_NEW = f'0.{str(int(num[1]) + 1)}.0' - elif VERSION_OLD.find('b') > -1: + elif VERSION_LAST_RELEASE.find('b') > -1: lastnum = num[2].split('b') lastnum[1] = str(int(lastnum[1]) + 1) VERSION_NEW = f'{num[0]}.{num[1]}.{lastnum[0]}b{lastnum[1]}' - elif VERSION_OLD.find('rc') > -1: + elif VERSION_LAST_RELEASE.find('rc') > -1: lastnum = num[2].split('rc') lastnum[1] = str(int(lastnum[1]) + 1) VERSION_NEW = f'{num[0]}.{num[1]}.{lastnum[0]}rc{lastnum[1]}' @@ -121,7 +113,10 @@ def edit_version(add_content): elif flag[2]: VERSION_NEW = f'{num[0]}.{num[1]}.{int(num[2]) + 1}' - list_in[i] = f'VERSION = "{VERSION_NEW}"\n' + for i in range(0, len(list_in)): + if list_in[i].find('VERSION ') > -1: + list_in[i] = f'VERSION = "{VERSION_NEW}"\n' + break with open(f'{path}/{file_name}', 'w') as file_out: file_out.writelines(list_in) From c7c6dad1c80657f3ec8ed19bd29c7bce8b55b528 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 30 Mar 2021 10:10:09 +0800 Subject: [PATCH 57/59] msrest>=0.5.0 => msrest>=0.6.21 --- scripts/auto_release/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 8af0b4a63411..b1916a2cf853 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -146,6 +146,7 @@ def edit_file_setup(): list_in = file_in.readlines() for i in range(0, len(list_in)): list_in[i] = list_in[i].replace('msrestazure>=0.4.32,<2.0.0', 'azure-mgmt-core>=1.2.0,<2.0.0') + list_in[i] = list_in[i].replace('msrest>=0.5.0', 'msrest>=0.6.21') with open(f'{path}/setup.py', 'w') as file_out: file_out.writelines(list_in) From b69303b67f0bd788d8f2e70201b725b7484d58c7 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Wed, 31 Mar 2021 17:30:24 +0800 Subject: [PATCH 58/59] dependency check --- scripts/auto_release/main.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b1916a2cf853..a5d76f35ce21 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -150,6 +150,17 @@ def edit_file_setup(): with open(f'{path}/setup.py', 'w') as file_out: file_out.writelines(list_in) + # avoid pipeline check fail + with open(f'shared_requirements.txt', 'r') as file_in: + list_in = file_in.readlines() + new_line = f'#override azure-mgmt-{SERVICE_NAME} msrest>=0.6.21' + for i in range(0, len(list_in)): + if list_in[i].find('new_line') > -1: + return + list_in.append(f'{new_line}\n') + with open(f'shared_requirements.txt', 'w') as file_out: + file_out.writelines(list_in) + def edit_file_readme(): path = f'sdk/{SDK_FOLDER}/azure-mgmt-{SERVICE_NAME}' @@ -284,6 +295,7 @@ def edit_useless_file(): def commit_test(): print_exec('git add sdk/') + print_exec('git add shared_requirements.txt') print_exec('git commit -m \"test"') print_exec('git push -f origin HEAD') my_print(f'== {SERVICE_NAME}(track{TRACK}) Automatic Release live test done !!! ==') From 272a7bfdf80900b025a6b32f08d493cfed037b39 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Wed, 31 Mar 2021 17:34:34 +0800 Subject: [PATCH 59/59] dependency check 2 --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a5d76f35ce21..f45cfe6288f7 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -295,7 +295,6 @@ def edit_useless_file(): def commit_test(): print_exec('git add sdk/') - print_exec('git add shared_requirements.txt') print_exec('git commit -m \"test"') print_exec('git push -f origin HEAD') my_print(f'== {SERVICE_NAME}(track{TRACK}) Automatic Release live test done !!! ==') @@ -359,6 +358,7 @@ def create_branch(): def commit_file(): print_exec('git add sdk/') + print_exec('git add shared_requirements.txt') print_exec('git commit -m \"version,CHANGELOG\"') print_exec('git push -f origin HEAD') my_print(f'== {SERVICE_NAME}(track{TRACK}) Automatic Release file-edit done !!! ==')