diff --git a/scripts/live_test/CLITest.yml b/scripts/live_test/CLITest.yml index ba9368144e3..b3d96b8bfbc 100644 --- a/scripts/live_test/CLITest.yml +++ b/scripts/live_test/CLITest.yml @@ -68,6 +68,8 @@ jobs: Target: container cosmosdb: Target: cosmosdb + databoxedge: + Target: databoxedge deploymentmanager: Target: deploymentmanager dla: @@ -142,6 +144,8 @@ jobs: Target: sqlvm storage: Target: storage + synapse: + Target: synapse util: Target: util vm: diff --git a/scripts/live_test/HISTORY.rst b/scripts/live_test/HISTORY.rst index 86b6adf31fb..9bfe3436702 100644 --- a/scripts/live_test/HISTORY.rst +++ b/scripts/live_test/HISTORY.rst @@ -3,6 +3,16 @@ Release History =============== +0.12.0 (4/15/2021) +++++++ + +* Add job databoxedge and synapse + +0.11.0 (1/18/2021) +++++++ + +* Add a job to automatically detect modules that have regression. + 0.10.0 (12/30/2020) ++++++ diff --git a/scripts/live_test/monitor.py b/scripts/live_test/monitor.py new file mode 100644 index 00000000000..65e6c2918ad --- /dev/null +++ b/scripts/live_test/monitor.py @@ -0,0 +1,94 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +# Monitor test results. Create issues when dropping. + +import sys +import requests + +DB_PWD = sys.argv[1] +GITHUB_TOKEN = sys.argv[2] + + +def main(): + data, regression_data = analyze_data() + create_issue(data, regression_data) + + +def analyze_data(): + import mysql.connector + # Connect + cnx = mysql.connector.connect(user='fey@clisqldbserver', + password=DB_PWD, + host='clisqldbserver.mysql.database.azure.com', + database='clidb') + cursor = cnx.cursor(buffered=True) + sql = 'select module from t2 group by module order by module;' + cursor.execute(sql) + modules = [] + for value in cursor: + if not value[0].startswith('ext-'): + modules.append(value[0]) + + print(modules) + + data = [] + regression_data = [] + for module in modules: + sql = "select t2.module, t2.pass, t2.fail, t2.rate, t1.container, t1.date, t1.time from t1 join t2 on t1.id = t2.ref_id where t2.module = '{}' and t1.branch = 'dev' and t1.target = '' and t1.repo = 'https://github.com/Azure/azure-cli.git' and t1.live = 1 order by t1.date desc, t1.time desc limit 2;" + cursor.execute(sql.format(module)) + new = old = None + for value in cursor: + data.append(value) + if new is None: + new = value + else: + old = value + # print(new) + # print(old) + if new[2] > old[2] or float(new[3].strip('%')) < float(old[3].strip('%')): + regression_data.append(new) + regression_data.append(old) + # print(new) + # print(old) + + # print(data) + + return data, regression_data + + +def create_issue(data, regression_data): + # Create Github issue + headers = { + 'Accept': 'application/vnd.github.v3+json' + } + issue_body = ''' +This issue is created by a program. +[Latest testing results of Azure CLI](https://clitestresultstac.blob.core.windows.net/latest/index.html) +[User Manual of Live Test Pipeline](https://microsoft-my.sharepoint.com/:w:/p/fey/EZGC9LwrN3RAscVS5ylG4HMBX9h7W0ZSA7CDrhXN5Lvx6g?e=V8HUmd) +[Upgrading API Versions in Azure CLI Live Test Pipeline](https://microsoft-my.sharepoint.com/:w:/p/fey/EcgPLHSkef9Mi14Rjx79N9sBvyVDO4b_V97BMcoI1HTq-A?e=Ioap3B) +[Power BI Report](https://msit.powerbi.com/groups/8de24d49-e97c-4672-9bfc-45fee0ec58f7/reports/65dfcfce-5d59-4dc9-8bc5-3726443c8fe1/ReportSection) + +The below table shows modules that have regression. Code owners, please pay attention. + +| Module | Pass | Fail | Pass rate | Detail | Date | Time | +| --- | --- | --- | --- | --- | --- | --- | +''' + + for entry in regression_data: + issue_body += '| {} | {} | {} | {} | [Link]({}) | {} | {} |\n'.format(entry[0], entry[1], entry[2], entry[3], entry[4], entry[5], entry[6]) + + request_body = { + 'title': 'Live test regression', + 'body': issue_body + } + r = requests.post('https://api.github.com/repos/Azure/azure-cli/issues', + headers=headers, auth=('user', GITHUB_TOKEN), json=request_body) + print(r.status_code) + print(r.content) + + +if __name__ == '__main__': + main() diff --git a/scripts/live_test/monitor.yml b/scripts/live_test/monitor.yml new file mode 100644 index 00000000000..2ad83f1a2db --- /dev/null +++ b/scripts/live_test/monitor.yml @@ -0,0 +1,20 @@ +trigger: + branches: + exclude: + - '*' + +jobs: +- job: Monitor + displayName: Monitor + pool: + vmImage: 'Ubuntu-18.04' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.8' + addToPath: true + architecture: 'x64' + - bash: | + pip install mysql-connector-python + pip install requests + python /home/vsts/work/1/s/scripts/live_test/monitor.py "$(DB_PWD)" "$(GITHUB_TOKEN)" \ No newline at end of file