From 0c8cb7bffe261bd76dae8e130a629d37235b42fe Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Tue, 23 Mar 2021 15:28:38 -0700 Subject: [PATCH 1/4] add perf tests --- .../tests/perfstress_tests/README.md | 44 ++++++++++++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/list_anomalies.py | 45 ++++++++++++++++++ .../tests/perfstress_tests/list_incidents.py | 46 +++++++++++++++++++ .../perfstress_tests/list_root_causes.py | 45 ++++++++++++++++++ 5 files changed, 180 insertions(+) create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/__init__.py create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..38190387c5fd --- /dev/null +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md @@ -0,0 +1,44 @@ +# Metrics advisor Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the `dev_requirements`. +Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7. + +### Setup for test resources + +These tests will run against a pre-configured metrics advisor service. The following environment variable will need to be set for the tests to access the live resources: +``` +AZURE_APP_CONFIG_CONNECTION_STRING= +``` + +### Setup for perf test runs + +```cmd +(env) ~/azure-ai-metricsadvisor> pip install -r dev_requirements.txt +(env) ~/azure-ai-metricsadvisor> pip install -e . +``` + +## Test commands + +When `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the current module for runable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature). + +```cmd +(env) ~/azure-ai-metricsadvisor> cd tests +(env) ~/azure-ai-metricsadvisor/tests> perfstress +``` +Using the `perfstress` command alone will list the available perf tests found. + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--warmup=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +## Example command +```cmd +(env) ~/azure-ai-metricsadvisor/tests> perfstress ListAnomaliesTest +(env) ~/azure-ai-metricsadvisor/tests> perfstress ListIncidentsTest +(env) ~/azure-ai-metricsadvisor/tests> perfstress ListRootCausesTest +``` diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/__init__.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py new file mode 100644 index 000000000000..20c039b1ac46 --- /dev/null +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py @@ -0,0 +1,45 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.ai.metricsadvisor import MetricsAdvisorClient as SyncClient, MetricsAdvisorKeyCredential +from azure.ai.metricsadvisor.aio import MetricsAdvisorClient as AsyncClient + + +class ListAnomaliesTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + service_endpoint = os.getenv("AZURE_METRICS_ADVISOR_ENDPOINT") + subscription_key = os.getenv("AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY") + api_key = os.getenv("AZURE_METRICS_ADVISOR_API_KEY") + self.anomaly_alert_configuration_id = os.getenv("AZURE_METRICS_ADVISOR_ANOMALY_ALERT_CONFIGURATION_ID") + self.alert_id = os.getenv("AZURE_METRICS_ADVISOR_ALERT_ID") + self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + + async def global_setup(self): + await super().global_setup() + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + ret = list(self.service_client.list_anomalies( + alert_configuration_id=self.anomaly_alert_configuration_id, + alert_id=self.alert_id, + )) + + async def run_async(self): + results = self.async_service_client.list_anomalies( + alert_configuration_id=self.anomaly_alert_configuration_id, + alert_id=self.alert_id, + ) + tolist = [] + async for result in results: + tolist.append(result) diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py new file mode 100644 index 000000000000..1f4334421604 --- /dev/null +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py @@ -0,0 +1,46 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import datetime +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.ai.metricsadvisor import MetricsAdvisorClient as SyncClient, MetricsAdvisorKeyCredential +from azure.ai.metricsadvisor.aio import MetricsAdvisorClient as AsyncClient + + +class ListIncidentsTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + service_endpoint = os.getenv("AZURE_METRICS_ADVISOR_ENDPOINT") + subscription_key = os.getenv("AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY") + api_key = os.getenv("AZURE_METRICS_ADVISOR_API_KEY") + self.anomaly_detection_configuration_id = os.getenv("AZURE_METRICS_ADVISOR_ANOMALY_DETECTION_CONFIGURATION_ID") + self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + + async def global_setup(self): + await super().global_setup() + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + ret = list(self.service_client.list_incidents( + detection_configuration_id=self.anomaly_detection_configuration_id, + start_time=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + end_time=datetime.datetime(2020, 10, 21, tzinfo=datetime.timezone.utc), + )) + + async def run_async(self): + results = self.async_service_client.list_incidents( + detection_configuration_id=self.anomaly_detection_configuration_id, + start_time=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + end_time=datetime.datetime(2020, 10, 21, tzinfo=datetime.timezone.utc), + ) + tolist = [] + async for result in results: + tolist.append(result) diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py new file mode 100644 index 000000000000..b98f21085464 --- /dev/null +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py @@ -0,0 +1,45 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.ai.metricsadvisor import MetricsAdvisorClient as SyncClient, MetricsAdvisorKeyCredential +from azure.ai.metricsadvisor.aio import MetricsAdvisorClient as AsyncClient + + +class ListRootCausesTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + service_endpoint = os.getenv("AZURE_METRICS_ADVISOR_ENDPOINT") + subscription_key = os.getenv("AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY") + api_key = os.getenv("AZURE_METRICS_ADVISOR_API_KEY") + self.anomaly_detection_configuration_id = os.getenv("AZURE_METRICS_ADVISOR_ANOMALY_DETECTION_CONFIGURATION_ID") + self.incident_id = os.getenv("AZURE_METRICS_ADVISOR_INCIDENT_ID") + self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + + async def global_setup(self): + await super().global_setup() + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + ret = list(self.service_client.list_incident_root_causes( + detection_configuration_id=self.anomaly_detection_configuration_id, + incident_id=self.incident_id, + )) + + async def run_async(self): + results = self.async_service_client.list_incident_root_causes( + detection_configuration_id=self.anomaly_detection_configuration_id, + incident_id=self.incident_id, + ) + tolist = [] + async for result in results: + tolist.append(result) From dabaa7418f28ba8091559744090f3e485dcf149b Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Tue, 23 Mar 2021 15:49:02 -0700 Subject: [PATCH 2/4] update --- .../tests/perfstress_tests/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md index 38190387c5fd..2eed5a1af978 100644 --- a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md @@ -7,7 +7,13 @@ Start be creating a new virtual environment for your perf tests. This will need These tests will run against a pre-configured metrics advisor service. The following environment variable will need to be set for the tests to access the live resources: ``` -AZURE_APP_CONFIG_CONNECTION_STRING= +AZURE_METRICS_ADVISOR_ENDPOINT= +AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY= +AZURE_METRICS_ADVISOR_API_KEY= +AZURE_METRICS_ADVISOR_ANOMALY_ALERT_CONFIGURATION_ID= +AZURE_METRICS_ADVISOR_ALERT_ID= +AZURE_METRICS_ADVISOR_ANOMALY_DETECTION_CONFIGURATION_ID= +AZURE_METRICS_ADVISOR_INCIDENT_ID= ``` ### Setup for perf test runs From 34b283e7e415b387d1d1c2b0a368f0dc395500d2 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Tue, 13 Apr 2021 09:09:35 -0700 Subject: [PATCH 3/4] update --- .../azure-ai-metricsadvisor/tests/perfstress_tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md index 2eed5a1af978..37e7f0d57a3b 100644 --- a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md @@ -5,7 +5,7 @@ Start be creating a new virtual environment for your perf tests. This will need ### Setup for test resources -These tests will run against a pre-configured metrics advisor service. The following environment variable will need to be set for the tests to access the live resources: +These tests will run against a pre-configured metrics advisor service(see [Setting up Resource for Perf test Guide](https://microsoft.sharepoint.com/teams/AzureDeveloperExperience/_layouts/15/Doc.aspx?sourcedoc={b489fb65-ef3e-410c-a69e-c23fccf5fcca}&action=edit&wd=target%28Untitled%20Section.one%7C93e23c77-5539-4c7a-9059-efa745a43f58%2FSetting%20up%20Resource%20for%20Perf%20test%20Guide%7C395ad307-9fd1-4971-bcd5-6d336dde77cd%2F%29&wdorigin=703) about how to setup the instance and ingest data). The following environment variable will need to be set for the tests to access the live resources: ``` AZURE_METRICS_ADVISOR_ENDPOINT= AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY= From b933810683a02c2a6eef216fc6beb3b6542925d5 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Tue, 13 Apr 2021 13:56:45 -0700 Subject: [PATCH 4/4] updates --- .../tests/perfstress_tests/list_anomalies.py | 14 ++++++-------- .../tests/perfstress_tests/list_incidents.py | 14 ++++++-------- .../tests/perfstress_tests/list_root_causes.py | 14 ++++++-------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py index 20c039b1ac46..b74459b48d3d 100644 --- a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py @@ -22,24 +22,22 @@ def __init__(self, arguments): self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) - async def global_setup(self): - await super().global_setup() - async def close(self): await self.async_service_client.close() await super().close() def run_sync(self): - ret = list(self.service_client.list_anomalies( + results = self.service_client.list_anomalies( alert_configuration_id=self.anomaly_alert_configuration_id, alert_id=self.alert_id, - )) + ) + for _ in results: + pass async def run_async(self): results = self.async_service_client.list_anomalies( alert_configuration_id=self.anomaly_alert_configuration_id, alert_id=self.alert_id, ) - tolist = [] - async for result in results: - tolist.append(result) + async for _ in results: + pass diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py index 1f4334421604..8604236cbb53 100644 --- a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py @@ -21,19 +21,18 @@ def __init__(self, arguments): self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) - async def global_setup(self): - await super().global_setup() - async def close(self): await self.async_service_client.close() await super().close() def run_sync(self): - ret = list(self.service_client.list_incidents( + results = self.service_client.list_incidents( detection_configuration_id=self.anomaly_detection_configuration_id, start_time=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), end_time=datetime.datetime(2020, 10, 21, tzinfo=datetime.timezone.utc), - )) + ) + for _ in results: + pass async def run_async(self): results = self.async_service_client.list_incidents( @@ -41,6 +40,5 @@ async def run_async(self): start_time=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), end_time=datetime.datetime(2020, 10, 21, tzinfo=datetime.timezone.utc), ) - tolist = [] - async for result in results: - tolist.append(result) + async for _ in results: + pass diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py index b98f21085464..911a24979586 100644 --- a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py @@ -22,24 +22,22 @@ def __init__(self, arguments): self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) - async def global_setup(self): - await super().global_setup() - async def close(self): await self.async_service_client.close() await super().close() def run_sync(self): - ret = list(self.service_client.list_incident_root_causes( + results = self.service_client.list_incident_root_causes( detection_configuration_id=self.anomaly_detection_configuration_id, incident_id=self.incident_id, - )) + ) + for _ in results: + pass async def run_async(self): results = self.async_service_client.list_incident_root_causes( detection_configuration_id=self.anomaly_detection_configuration_id, incident_id=self.incident_id, ) - tolist = [] - async for result in results: - tolist.append(result) + async for _ in results: + pass