From 608dcb090f877b28a681428937b6d659f18dc430 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 19 Apr 2021 19:34:45 -0700 Subject: [PATCH 01/90] add sync and async version of RecordedByProxy. Grokking storage testcase setup/teardown --- .../tests/_shared/testcase.py | 12 +- .../tests/test_blob_client.py | 4 +- .../devtools_testutils/__init__.py | 2 + .../devtools_testutils/proxy_testcase.py | 162 ++++++++++++++++++ 4 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py diff --git a/sdk/storage/azure-storage-blob/tests/_shared/testcase.py b/sdk/storage/azure-storage-blob/tests/_shared/testcase.py index 8d203eb6db00..15775748c7ca 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/testcase.py +++ b/sdk/storage/azure-storage-blob/tests/_shared/testcase.py @@ -25,6 +25,7 @@ import random import re import logging +import pdb from devtools_testutils import ( AzureMgmtTestCase, AzureMgmtPreparer, @@ -53,10 +54,8 @@ import pytest - +ENABLE_LOGGING = True LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s' -os.environ['AZURE_STORAGE_ACCOUNT_NAME'] = STORAGE_ACCOUNT_NAME -os.environ['AZURE_STORAGE_ACCOUNT_KEY'] = STORAGE_ACCOUNT_KEY os.environ['AZURE_TEST_RUN_LIVE'] = os.environ.get('AZURE_TEST_RUN_LIVE', None) or RUN_IN_LIVE os.environ['AZURE_SKIP_LIVE_RECORDING'] = os.environ.get('AZURE_SKIP_LIVE_RECORDING', None) or SKIP_LIVE_RECORDING @@ -398,9 +397,11 @@ def storage_account(): try: if i_need_to_create_rg: + logging.info("I NEED TO CREATE RG") rg_name, rg_kwargs = rg_preparer._prepare_create_resource(test_case) rg = rg_kwargs['resource_group'] else: + logging.info("I DONT NEED TO CREATE RG") rg_name = existing_rg_name or "no_rg_needed" rg = FakeResource( name=rg_name, @@ -408,9 +409,10 @@ def storage_account(): ) StorageTestCase._RESOURCE_GROUP = rg + try: if got_storage_info_from_env: - + logging.info("EXISTING INFO IN ENV") if storage_connection_string: storage_connection_string_parts = dict([ part.split('=', 1) @@ -419,6 +421,7 @@ def storage_account(): storage_account = None if existing_storage_name: + logging.info("EXISTING NAME IS {}".format(existing_storage_name)) storage_name = existing_storage_name storage_account = StorageAccount( location=location, @@ -472,6 +475,7 @@ def build_service_endpoint(service): storage_key = storage_connection_string_parts["AccountKey"] else: + logging.info("CREATE STORAGE ACCOUNT") storage_name, storage_kwargs = storage_preparer._prepare_create_resource(test_case, **rg_kwargs) storage_account = storage_kwargs['storage_account'] storage_key = storage_kwargs['storage_account_key'] diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_client.py b/sdk/storage/azure-storage-blob/tests/test_blob_client.py index 08e15585366c..5417a902f34b 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_client.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_client.py @@ -15,7 +15,7 @@ ContainerClient, BlobClient, ) -from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer +from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer, RecordedByProxy from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer #from azure.storage.common import TokenCredential @@ -48,9 +48,9 @@ def validate_standard_account_endpoints(self, service, url_type, name, storage_a # --Direct Parameters Test Cases -------------------------------------------- @GlobalStorageAccountPreparer() + @RecordedByProxy def test_create_service_with_key(self, resource_group, location, storage_account, storage_account_key): # Arrange - for client, url in SERVICES.items(): # Act service = client( diff --git a/tools/azure-sdk-tools/devtools_testutils/__init__.py b/tools/azure-sdk-tools/devtools_testutils/__init__.py index 4f90bd199676..548465f68739 100644 --- a/tools/azure-sdk-tools/devtools_testutils/__init__.py +++ b/tools/azure-sdk-tools/devtools_testutils/__init__.py @@ -13,6 +13,7 @@ ) from .keyvault_preparer import KeyVaultPreparer from .powershell_preparer import PowerShellPreparer +from .proxy_testcase import RecordedByProxy __all__ = [ "AzureMgmtTestCase", @@ -29,4 +30,5 @@ "RandomNameResourceGroupPreparer", "CachedResourceGroupPreparer", "PowerShellPreparer", + "RecordedByProxy" ] diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py new file mode 100644 index 000000000000..fd7b42f78005 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -0,0 +1,162 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import functools +import pdb +import os +import requests +from contextlib import contextmanager + +# the functions we patch +from azure.core.pipeline.transport import RequestsTransport, AsyncioRequestsTransport + +# the trimming function to clean up incoming arguments to the test function we are wrapping +from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function + +# defaults +PROXY_URL = "https://localhost:5001" +RECORDING_START_URL = "{}/record/start".format(PROXY_URL) +RECORDING_STOP_URL = "{}/record/stop".format(PROXY_URL) +PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL) +PLAYBACK_STOP_URL = "{}/playback/stop".format(PROXY_URL) + +# TODO, create a pytest scope="session" implementation that can be added to a fixture such that that can +# unit tests can startup/shutdown the local test proxy + +@contextmanager +def patch_requests_func(request_transform): + original_func = RequestsTransport.send + + def combined_call(*args, **kwargs): + adjusted_args, adjusted_kwargs = request_transform(*args,**kwargs) + return original_func(*adjusted_args, **adjusted_kwargs) + + RequestsTransport.send = combined_call + yield None + + RequestsTransport.send = original_func + +@contextmanager +def patch_requests_func_async(request_transform): + original_func = AsyncioRequestsTransport.send + + async def combined_call(*args, **kwargs): + adjusted_args, adjusted_kwargs = request_transform(*args,**kwargs) + return await original_func(*adjusted_args, **adjusted_kwargs) + + AsyncioRequestsTransport.send = combined_call + yield None + + AsyncioRequestsTransport.send = original_func + +def get_test_id(): + # pytest sets the current running test in an environment variable + return os.getenv('PYTEST_CURRENT_TEST').split(" ")[0].replace("::",".") + +def start_record_or_playback(test_id): + if os.getenv("AZURE_RECORD_MODE") == "record": + result = requests.post( + RECORDING_START_URL, headers={"x-recording-file": test_id}, verify=False + ) + recording_id = result.headers["x-recording-id"] + elif os.getenv("AZURE_RECORD_MODE") == "playback": + result = requests.post( + PLAYBACK_START_URL, + headers={"x-recording-file": test_id, "x-recording-id": recording_id}, + verify=False, + ) + recording_id = result.headers["x-recording-id"] + return recording_id + +def stop_record_or_playback(test_id, recording_id): + if os.getenv("AZURE_RECORD_MODE") == "record": + result = requests.post( + RECORDING_STOP_URL, + headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, + verify=False, + ) + elif os.getenv("AZURE_RECORD_MODE") == "playback": + result = requests.post( + PLAYBACK_STOP_URL, + headers={"x-recording-file": test_id, "x-recording-id": recording_id}, + verify=False, + ) + +def transform_request(request, recording_id): + upstream_url = request.url.replace("//text", "/text") + headers = request.headers + + # quiet passthrough if neither are set + if os.getenv("AZURE_RECORD_MODE") == "record" or os.getenv("AZURE_RECORD_MODE") == "playback": + headers["x-recording-upstream-base-uri"] = upstream_url + headers["x-recording-id"] = recording_id + headers["x-recording-mode"] = os.getenv("AZURE_RECORD_MODE") + request.url = PROXY_URL + +def RecordedByProxyAsync(func): + @functools.wraps(func) + async def record_wrap(*args, **kwargs): + test_id = get_test_id() + recording_id = get_recording_id(test_id) + + def transform_args(*args, **kwargs): + copied_positional_args = list(args) + request = copied_positional_args[1] + + # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are + # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get + # a recommendation for how to get a valid SSL Cert for localhost + kwargs["connection_verify"] = False + + transform_request(request, recording_id) + + return tuple(copied_positional_args), kwargs + + trimmed_kwargs = {k:v for k,v in kwargs.items()} + trim_kwargs_from_test_function(func, trimmed_kwargs) + + # this ensures that within this scope, we've monkeypatched the send functionality + with patch_requests_func_async(transform_args): + # call the modified function. + value = await func(*args, **trimmed_kwargs) + + stop_record_or_playback(test_id, recording_id) + return value + + return record_wrap + +def RecordedByProxy(func): + @functools.wraps(func) + def record_wrap(*args, **kwargs): + test_id = get_test_id() + recording_id = start_record_or_playback(test_id) + + def transform_args(*args, **kwargs): + copied_positional_args = list(args) + request = copied_positional_args[1] + + # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are + # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get + # a recommendation for how to get a valid SSL Cert for localhost + kwargs["connection_verify"] = False + + transform_request(request, recording_id) + + return tuple(copied_positional_args), kwargs + + trimmed_kwargs = {k:v for k,v in kwargs.items()} + trim_kwargs_from_test_function(func, trimmed_kwargs) + + # this ensures that within this scope, we've monkeypatched the send functionality + with patch_requests_func(transform_args): + # call the modified function. + value = func(*args, **trimmed_kwargs) + + stop_record_or_playback(test_id, recording_id) + + return value + + return record_wrap \ No newline at end of file From 9e319a098d34083a218029076e9ec59b07e4e66a Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 21 Apr 2021 21:36:26 -0700 Subject: [PATCH 02/90] passing SHA to the test-proxy server --- .../devtools_testutils/proxy_testcase.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index fd7b42f78005..72edb3edf02c 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -10,6 +10,9 @@ import requests from contextlib import contextmanager +import subprocess + + # the functions we patch from azure.core.pipeline.transport import RequestsTransport, AsyncioRequestsTransport @@ -25,6 +28,7 @@ # TODO, create a pytest scope="session" implementation that can be added to a fixture such that that can # unit tests can startup/shutdown the local test proxy +# this should also fire the admin mapping updates, and start/end the session for commiting recording updates @contextmanager def patch_requests_func(request_transform): @@ -56,10 +60,16 @@ def get_test_id(): # pytest sets the current running test in an environment variable return os.getenv('PYTEST_CURRENT_TEST').split(" ")[0].replace("::",".") +def get_current_sha(): + result = subprocess.check_output(["git", "rev-parse", "HEAD"]) + + #TODO: is this compatible with py27? + return result.decode('utf-8').strip() + def start_record_or_playback(test_id): if os.getenv("AZURE_RECORD_MODE") == "record": result = requests.post( - RECORDING_START_URL, headers={"x-recording-file": test_id}, verify=False + RECORDING_START_URL, headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha() }, verify=False ) recording_id = result.headers["x-recording-id"] elif os.getenv("AZURE_RECORD_MODE") == "playback": From 99535674d610e1bb30617e6eb0e07a4355119ae5 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Fri, 7 May 2021 11:57:36 -0400 Subject: [PATCH 03/90] have new test class, proxy recorder is not starting --- .../azure-data-tables/tests/test_table.py | 15 +- .../scenario_tests/preparers.py | 9 +- .../devtools_testutils/__init__.py | 2 + .../azure_recorded_testcase.py | 271 ++++++++++++++++++ .../devtools_testutils/powershell_preparer.py | 8 +- 5 files changed, 292 insertions(+), 13 deletions(-) create mode 100644 tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index 2d0baa8a5d8c..23c98d91638f 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -6,13 +6,12 @@ # license information. # -------------------------------------------------------------------------- import pytest - -import sys -import locale -import os from datetime import datetime, timedelta -from devtools_testutils import AzureTestCase +from devtools_testutils import ( + AzureRecordedTestCase, + RecordedByProxy +) from azure.data.tables import ( ResourceTypes, @@ -50,7 +49,7 @@ # ------------------------------------------------------------------------------ -class StorageTableTest(AzureTestCase, TableTestCase): +class TestStorageTable(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _get_table_reference(self, prefix=TEST_TABLE_PREFIX): @@ -106,6 +105,7 @@ def test_create_properties(self, tables_storage_account_name, tables_primary_sto ps = ts.get_service_properties() ts.delete_table(table_name) + @RecordedByProxy @tables_decorator def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key): # # Arrange @@ -121,6 +121,7 @@ def test_create_table(self, tables_storage_account_name, tables_primary_storage_ assert created.table_name == table_name ts.delete_table(table_name) + @RecordedByProxy @tables_decorator def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange @@ -172,6 +173,7 @@ def test_query_tables_per_page(self, tables_storage_account_name, tables_primary if self.is_live: self.sleep(10) # wait for tables to be deleted before proceeding + @RecordedByProxy @tables_decorator def test_create_table_if_exists(self, tables_storage_account_name, tables_primary_storage_account_key): account_url = self.account_url(tables_storage_account_name, "table") @@ -186,6 +188,7 @@ def test_create_table_if_exists(self, tables_storage_account_name, tables_primar assert t0.table_name == t1.table_name ts.delete_table(table_name) + @RecordedByProxy @tables_decorator def test_create_table_if_exists_new_table(self, tables_storage_account_name, tables_primary_storage_account_key): account_url = self.account_url(tables_storage_account_name, "table") diff --git a/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py b/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py index 50915329e9a3..1f9c195c8847 100644 --- a/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py +++ b/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py @@ -132,10 +132,11 @@ def _preparer_wrapper(test_class_instance, **kwargs): AbstractPreparer._resource_cache[aggregate_cache_key] = AbstractPreparer.ResourceCacheEntry(resource_name, kwargs, self) if test_class_instance.is_live: - test_class_instance.scrubber.register_name_pair( - resource_name, - self.moniker - ) + if hasattr(test_class_instance, "scrubber"): + test_class_instance.scrubber.register_name_pair( + resource_name, + self.moniker + ) # We shouldn't trim the same kwargs that we use for deletion, # we may remove some of the variables we needed to do the delete. diff --git a/tools/azure-sdk-tools/devtools_testutils/__init__.py b/tools/azure-sdk-tools/devtools_testutils/__init__.py index 548465f68739..20e46ab539a4 100644 --- a/tools/azure-sdk-tools/devtools_testutils/__init__.py +++ b/tools/azure-sdk-tools/devtools_testutils/__init__.py @@ -1,4 +1,5 @@ from .mgmt_testcase import AzureMgmtTestCase, AzureMgmtPreparer +from .azure_recorded_testcase import AzureRecordedTestCase from .azure_testcase import AzureTestCase, is_live, get_region_override from .resource_testcase import ( FakeResource, @@ -16,6 +17,7 @@ from .proxy_testcase import RecordedByProxy __all__ = [ + "AzureRecordedTestCase", "AzureMgmtTestCase", "AzureMgmtPreparer", "FakeResource", diff --git a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py new file mode 100644 index 000000000000..d6b0f87d27a7 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py @@ -0,0 +1,271 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import functools +import inspect +import logging +import os.path +import sys +import time +import zlib + +try: + from inspect import getfullargspec as get_arg_spec +except ImportError: + from inspect import getargspec as get_arg_spec + +try: + from urllib.parse import quote +except ImportError: + from urllib2 import quote # type: ignore + +import pytest +from dotenv import load_dotenv, find_dotenv + +from azure_devtools.scenario_tests import ( + ReplayableTest, + AzureTestError, + GeneralNameReplacer, + RequestUrlNormalizer, + AuthenticationMetadataFilter, + OAuthRequestResponsesFilter, +) +from azure_devtools.scenario_tests.config import TestConfig +from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function + +from .config import TEST_SETTING_FILENAME +from . import mgmt_settings_fake as fake_settings +from .azure_testcase import is_live, _is_autorest_v3, get_resource_name, get_qualified_method_name + +try: + # Try to import the AsyncFakeCredential, if we cannot assume it is Python 2 + from .fake_async_credential import AsyncFakeCredential +except SyntaxError: + pass + + +from dotenv import load_dotenv, find_dotenv + +load_dotenv(find_dotenv()) +print(os.environ["AZURE_RECORD_MODE"]) + +class AzureRecordedTestCase(object): + + @property + def settings(self): + if self.is_live: + if self._real_settings: + return self._real_settings + else: + raise AzureTestError( + "Need a mgmt_settings_real.py file to run tests live." + ) + else: + return self._fake_settings + + def _load_settings(self): + try: + from . import mgmt_settings_real as real_settings + + return fake_settings, real_settings + except ImportError: + return fake_settings, None + + @property + def is_live(self): + return is_live() + + @property + def qualified_test_name(self): + return get_qualified_method_name(self, "method_name") + + def is_playback(self): + return not self.is_live + + def get_settings_value(self, key): + key_value = os.environ.get("AZURE_" + key, None) + + if ( + key_value + and self._real_settings + and getattr(self._real_settings, key) != key_value + ): + raise ValueError( + "You have both AZURE_{key} env variable and mgmt_settings_real.py for {key} to different values".format( + key=key + ) + ) + + if not key_value: + try: + key_value = getattr(self.settings, key) + except Exception: + print("Could not get {}".format(key)) + raise + return key_value + + def get_credential(self, client_class, **kwargs): + + tenant_id = os.environ.get( + "AZURE_TENANT_ID", getattr(self._real_settings, "TENANT_ID", None) + ) + client_id = os.environ.get( + "AZURE_CLIENT_ID", getattr(self._real_settings, "CLIENT_ID", None) + ) + secret = os.environ.get( + "AZURE_CLIENT_SECRET", getattr(self._real_settings, "CLIENT_SECRET", None) + ) + is_async = kwargs.pop("is_async", False) + + if tenant_id and client_id and secret and self.is_live: + if _is_autorest_v3(client_class): + # Create azure-identity class + from azure.identity import ClientSecretCredential + + if is_async: + from azure.identity.aio import ClientSecretCredential + return ClientSecretCredential( + tenant_id=tenant_id, client_id=client_id, client_secret=secret + ) + else: + # Create msrestazure class + from msrestazure.azure_active_directory import ( + ServicePrincipalCredentials, + ) + + return ServicePrincipalCredentials( + tenant=tenant_id, client_id=client_id, secret=secret + ) + else: + if _is_autorest_v3(client_class): + if is_async: + if self.is_live: + raise ValueError( + "Async live doesn't support mgmt_setting_real, please set AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET" + ) + return AsyncFakeCredential() + else: + return self.settings.get_azure_core_credentials() + else: + return self.settings.get_credentials() + + def create_client_from_credential(self, client_class, credential, **kwargs): + + # Real client creation + # TODO decide what is the final argument for that + # if self.is_playback(): + # kwargs.setdefault("polling_interval", 0) + if _is_autorest_v3(client_class): + kwargs.setdefault("logging_enable", True) + client = client_class(credential=credential, **kwargs) + else: + client = client_class(credentials=credential, **kwargs) + + if self.is_playback(): + try: + client._config.polling_interval = ( + 0 # FIXME in azure-mgmt-core, make this a kwargs + ) + except AttributeError: + pass + + if hasattr(client, "config"): # Autorest v2 + if self.is_playback(): + client.config.long_running_operation_timeout = 0 + client.config.enable_http_logger = True + return client + + def create_basic_client(self, client_class, **kwargs): + """ DO NOT USE ME ANYMORE.""" + logger = logging.getLogger() + logger.warning( + "'create_basic_client' will be deprecated in the future. It is recommended that you use \ + 'get_credential' and 'create_client_from_credential' to create your client." + ) + + credentials = self.get_credential(client_class) + return self.create_client_from_credential(client_class, credentials, **kwargs) + + def create_random_name(self, name): + return get_resource_name(name, self.qualified_test_name.encode()) + + def get_resource_name(self, name): + """Alias to create_random_name for back compatibility.""" + return self.create_random_name(name) + + def get_replayable_random_resource_name(self, name): + """In a replay scenario, (is not live) gives the static moniker. In the random scenario, gives generated name.""" + if self.is_live: + created_name = self.create_random_name(name) + self.scrubber.register_name_pair(created_name, name) + return name + + def get_preparer_resource_name(self, prefix): + """Random name generation for use by preparers. + + If prefix is a blank string, use the fully qualified test name instead. + This is what legacy tests do for resource groups.""" + return self.get_resource_name( + prefix #or self.qualified_test_name.replace(".", "_") + ) + + @staticmethod + def await_prepared_test(test_fn): + """Synchronous wrapper for async test methods. Used to avoid making changes + upstream to AbstractPreparer, which only awaits async tests that use preparers. + (Add @AzureTestCase.await_prepared_test decorator to async tests without preparers) + + # Note: this will only be needed so long as we maintain unittest.TestCase in our + test-class inheritance chain. + """ + + if sys.version_info < (3, 5): + raise ImportError("Async wrapper is not needed for Python 2.7 code.") + + import asyncio + + @functools.wraps(test_fn) + def run(test_class_instance, *args, **kwargs): + trim_kwargs_from_test_function(test_fn, kwargs) + loop = asyncio.get_event_loop() + return loop.run_until_complete(test_fn(test_class_instance, **kwargs)) + + return run + + def sleep(self, seconds): + if self.is_live: + time.sleep(seconds) + + def generate_sas(self, *args, **kwargs): + sas_func = args[0] + sas_func_pos_args = args[1:] + + fake_value = kwargs.pop("fake_value", "fake_token_value") + token = sas_func(*sas_func_pos_args, **kwargs) + + fake_token = self._create_fake_token(token, fake_value) + + if self.is_live: + return token + return fake_token + + def _create_fake_token(self, token, fake_value): + parts = token.split("&") + + for idx, part in enumerate(parts): + if part.startswith("sig"): + key = part.split("=") + key[1] = fake_value + parts[idx] = "=".join(key) + elif part.startswith("st"): + key = part.split("=") + key[1] = "start" + parts[idx] = "=".join(key) + elif part.startswith("se"): + key = part.split("=") + key[1] = "end" + parts[idx] = "=".join(key) + + return "&".join(parts) diff --git a/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py b/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py index 6e36436426b2..02c8f199adf6 100644 --- a/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py +++ b/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py @@ -71,9 +71,11 @@ def create_resource(self, name, **kwargs): scrubbed_value = self.fake_values[key] if scrubbed_value: self.real_values[key.lower()] = os.environ[key.upper()] - self.test_class_instance.scrubber.register_name_pair( - self.real_values[key.lower()], scrubbed_value - ) + # Adding this for new proxy testcase + if hasattr(self.test_class_instance, "scrubber"): + self.test_class_instance.scrubber.register_name_pair( + self.real_values[key.lower()], scrubbed_value + ) else: template = 'To pass a live ID you must provide the scrubbed value for recordings to \ prevent secrets from being written to files. {} was not given. For example: \ From 402eec853aea440376170f72a482ec323dc7f86b Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Fri, 7 May 2021 15:15:49 -0400 Subject: [PATCH 04/90] added more properties, to the testcase class --- .../azure-data-tables/tests/test_table.py | 8 ++--- .../azure_recorded_testcase.py | 36 +++++++------------ 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index 23c98d91638f..fc63f04efa71 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -105,8 +105,8 @@ def test_create_properties(self, tables_storage_account_name, tables_primary_sto ps = ts.get_service_properties() ts.delete_table(table_name) - @RecordedByProxy @tables_decorator + @RecordedByProxy def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key): # # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -121,8 +121,8 @@ def test_create_table(self, tables_storage_account_name, tables_primary_storage_ assert created.table_name == table_name ts.delete_table(table_name) - @RecordedByProxy @tables_decorator + @RecordedByProxy def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -173,8 +173,8 @@ def test_query_tables_per_page(self, tables_storage_account_name, tables_primary if self.is_live: self.sleep(10) # wait for tables to be deleted before proceeding - @RecordedByProxy @tables_decorator + @RecordedByProxy def test_create_table_if_exists(self, tables_storage_account_name, tables_primary_storage_account_key): account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) @@ -188,8 +188,8 @@ def test_create_table_if_exists(self, tables_storage_account_name, tables_primar assert t0.table_name == t1.table_name ts.delete_table(table_name) - @RecordedByProxy @tables_decorator + @RecordedByProxy def test_create_table_if_exists_new_table(self, tables_storage_account_name, tables_primary_storage_account_key): account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) diff --git a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py index d6b0f87d27a7..879727490072 100644 --- a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py @@ -4,35 +4,15 @@ # license information. # -------------------------------------------------------------------------- import functools -import inspect import logging +import os import os.path import sys import time -import zlib -try: - from inspect import getfullargspec as get_arg_spec -except ImportError: - from inspect import getargspec as get_arg_spec - -try: - from urllib.parse import quote -except ImportError: - from urllib2 import quote # type: ignore - -import pytest from dotenv import load_dotenv, find_dotenv -from azure_devtools.scenario_tests import ( - ReplayableTest, - AzureTestError, - GeneralNameReplacer, - RequestUrlNormalizer, - AuthenticationMetadataFilter, - OAuthRequestResponsesFilter, -) -from azure_devtools.scenario_tests.config import TestConfig +from azure_devtools.scenario_tests import AzureTestError from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function from .config import TEST_SETTING_FILENAME @@ -46,8 +26,6 @@ pass -from dotenv import load_dotenv, find_dotenv - load_dotenv(find_dotenv()) print(os.environ["AZURE_RECORD_MODE"]) @@ -81,6 +59,16 @@ def is_live(self): def qualified_test_name(self): return get_qualified_method_name(self, "method_name") + @property + def in_recording(self): + return os.getenv("AZURE_RECORD_MODE") == "record" + + # TODO: This needs to be removed, recording processors are handled on the proxy side, but + # this is needed for the preparers + @property + def recording_processors(self): + return [] + def is_playback(self): return not self.is_live From c52329e791f1e76e335396f6cf40870ef4197b43 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Mon, 10 May 2021 14:15:06 -0400 Subject: [PATCH 05/90] https -> http --- .../devtools_testutils/proxy_testcase.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 72edb3edf02c..d8e34360c584 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -20,13 +20,13 @@ from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function # defaults -PROXY_URL = "https://localhost:5001" +PROXY_URL = "http://localhost:5001" RECORDING_START_URL = "{}/record/start".format(PROXY_URL) RECORDING_STOP_URL = "{}/record/stop".format(PROXY_URL) PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL) PLAYBACK_STOP_URL = "{}/playback/stop".format(PROXY_URL) -# TODO, create a pytest scope="session" implementation that can be added to a fixture such that that can +# TODO, create a pytest scope="session" implementation that can be added to a fixture such that that can # unit tests can startup/shutdown the local test proxy # this should also fire the admin mapping updates, and start/end the session for commiting recording updates @@ -116,8 +116,8 @@ def transform_args(*args, **kwargs): copied_positional_args = list(args) request = copied_positional_args[1] - # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are - # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get + # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are + # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get # a recommendation for how to get a valid SSL Cert for localhost kwargs["connection_verify"] = False @@ -148,8 +148,8 @@ def transform_args(*args, **kwargs): copied_positional_args = list(args) request = copied_positional_args[1] - # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are - # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get + # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are + # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get # a recommendation for how to get a valid SSL Cert for localhost kwargs["connection_verify"] = False From 94dc285471b153a642bd08810f1ce63b15f73f18 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Mon, 10 May 2021 17:25:52 -0400 Subject: [PATCH 06/90] works for proxy --- .../azure-data-tables/tests/test_table.py | 66 ++++++++++--------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index fc63f04efa71..c4475a8f4693 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -434,6 +434,7 @@ def test_set_table_acl_too_many_ids(self, tables_storage_account_name, tables_pr @pytest.mark.live_test_only @tables_decorator + @RecordedByProxy def test_account_sas(self, tables_storage_account_name, tables_primary_storage_account_key): # SAS URL is calculated from storage key, so this test runs live only @@ -441,41 +442,46 @@ def test_account_sas(self, tables_storage_account_name, tables_primary_storage_a account_url = self.account_url(tables_storage_account_name, "table") tsc = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) - table = self._create_table(tsc) - try: - entity = { - 'PartitionKey': u'test', - 'RowKey': u'test1', - 'text': u'hello', - } - table.upsert_entity(mode=UpdateMode.MERGE, entity=entity) - - entity['RowKey'] = u'test2' - table.upsert_entity(mode=UpdateMode.MERGE, entity=entity) + # table = self._create_table(tsc) + # print(table.table_name) + # try: + table = tsc.get_table_client(u"pytablesync669b08d7") + entity = { + 'PartitionKey': u'test', + 'RowKey': u'test1', + 'text': u'hello', + } + table.upsert_entity(mode=UpdateMode.MERGE, entity=entity) + + entity['RowKey'] = u'test2' + table.upsert_entity(mode=UpdateMode.MERGE, entity=entity) + + token = generate_account_sas( + tables_primary_storage_account_key, + resource_types=ResourceTypes(object=True), + permission=AccountSasPermissions(read=True), + expiry=datetime.utcnow() + timedelta(hours=1), + start=datetime.utcnow() - timedelta(minutes=1), + ) - token = generate_account_sas( - tables_primary_storage_account_key, - resource_types=ResourceTypes(object=True), - permission=AccountSasPermissions(read=True), - expiry=datetime.utcnow() + timedelta(hours=1), - start=datetime.utcnow() - timedelta(minutes=1), - ) - - account_url = self.account_url(tables_storage_account_name, "table") + account_url = self.account_url(tables_storage_account_name, "table") - service = self.create_client_from_credential(TableServiceClient, token, endpoint=account_url) + service = self.create_client_from_credential(TableServiceClient, token, endpoint=account_url) - # Act + # Act - sas_table = service.get_table_client(table.table_name) - entities = list(sas_table.list_entities()) + sas_table = service.get_table_client(table.table_name) + entities = list(sas_table.list_entities()) - # Assert - assert len(entities) == 2 - assert entities[0]['text'] == u'hello' - assert entities[1]['text'] == u'hello' - finally: - self._delete_table(table=table, ts=tsc) + # Assert + assert len(entities) == 2 + assert entities[0]['text'] == u'hello' + assert entities[1]['text'] == u'hello' + # finally: + # self._delete_table(table=table, ts=tsc) + + # for entity in table.list_entities(): + # table.delete_entity(entity["PartitionKey"], entity["RowKey"]) class TestTablesUnitTest(TableTestCase): From 004118eb0a54ca9b4108186cb1ef385b3b1f27b1 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Tue, 11 May 2021 13:52:43 -0400 Subject: [PATCH 07/90] adding more tests to proxy --- .../azure-data-tables/tests/test_table.py | 80 +++++++++++-------- .../devtools_testutils/proxy_testcase.py | 15 +++- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index c4475a8f4693..c58395941496 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -5,6 +5,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +from devtools_testutils.proxy_testcase import RecordedByProxyAsync import pytest from datetime import datetime, timedelta @@ -83,6 +84,7 @@ def _delete_all_tables(self, ts): # --Test cases for tables -------------------------------------------------- @tables_decorator + @RecordedByProxy def test_create_properties(self, tables_storage_account_name, tables_primary_storage_account_key): # # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -142,6 +144,7 @@ def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_pr ts.delete_table(table_name) @tables_decorator + @RecordedByProxy def test_query_tables_per_page(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -202,6 +205,7 @@ def test_create_table_if_exists_new_table(self, tables_storage_account_name, tab ts.delete_table(table_name) @tables_decorator + @RecordedByProxy def test_query_tables(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -224,6 +228,7 @@ def test_query_tables(self, tables_storage_account_name, tables_primary_storage_ self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator + @RecordedByProxy def test_query_tables_with_filter(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -248,6 +253,7 @@ def test_query_tables_with_filter(self, tables_storage_account_name, tables_prim self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator + @RecordedByProxy def test_query_tables_with_num_results(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange prefix = 'listtable' @@ -277,6 +283,7 @@ def test_query_tables_with_num_results(self, tables_storage_account_name, tables self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator + @RecordedByProxy def test_query_tables_with_marker(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -307,6 +314,7 @@ def test_query_tables_with_marker(self, tables_storage_account_name, tables_prim self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator + @RecordedByProxy def test_delete_table_with_existing_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -322,6 +330,7 @@ def test_delete_table_with_existing_table(self, tables_storage_account_name, tab assert len(existing) == 0 @tables_decorator + @RecordedByProxy def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -330,6 +339,7 @@ def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_storag ts.delete_table(table_name) @tables_decorator + @RecordedByProxy def test_get_table_acl(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange url = self.account_url(tables_storage_account_name, "table") @@ -347,6 +357,7 @@ def test_get_table_acl(self, tables_storage_account_name, tables_primary_storage ts.delete_table(table.table_name) @tables_decorator + @RecordedByProxy def test_set_table_acl_with_empty_signed_identifiers(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -366,6 +377,7 @@ def test_set_table_acl_with_empty_signed_identifiers(self, tables_storage_accoun ts.delete_table(table.table_name) @tables_decorator + @RecordedByProxy def test_set_table_acl_with_empty_signed_identifier(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -388,6 +400,7 @@ def test_set_table_acl_with_empty_signed_identifier(self, tables_storage_account ts.delete_table(table.table_name) @tables_decorator + @RecordedByProxy def test_set_table_acl_with_signed_identifiers(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -413,6 +426,7 @@ def test_set_table_acl_with_signed_identifiers(self, tables_storage_account_name ts.delete_table(table.table_name) @tables_decorator + @RecordedByProxy def test_set_table_acl_too_many_ids(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -442,46 +456,42 @@ def test_account_sas(self, tables_storage_account_name, tables_primary_storage_a account_url = self.account_url(tables_storage_account_name, "table") tsc = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) - # table = self._create_table(tsc) - # print(table.table_name) - # try: - table = tsc.get_table_client(u"pytablesync669b08d7") - entity = { - 'PartitionKey': u'test', - 'RowKey': u'test1', - 'text': u'hello', - } - table.upsert_entity(mode=UpdateMode.MERGE, entity=entity) - - entity['RowKey'] = u'test2' - table.upsert_entity(mode=UpdateMode.MERGE, entity=entity) - - token = generate_account_sas( - tables_primary_storage_account_key, - resource_types=ResourceTypes(object=True), - permission=AccountSasPermissions(read=True), - expiry=datetime.utcnow() + timedelta(hours=1), - start=datetime.utcnow() - timedelta(minutes=1), - ) + table = self._create_table(tsc) + try: + table = tsc.get_table_client(u"pytablesync669b08d7") + entity = { + 'PartitionKey': u'test', + 'RowKey': u'test1', + 'text': u'hello', + } + table.upsert_entity(mode=UpdateMode.MERGE, entity=entity) - account_url = self.account_url(tables_storage_account_name, "table") + entity['RowKey'] = u'test2' + table.upsert_entity(mode=UpdateMode.MERGE, entity=entity) - service = self.create_client_from_credential(TableServiceClient, token, endpoint=account_url) + token = generate_account_sas( + tables_primary_storage_account_key, + resource_types=ResourceTypes(object=True), + permission=AccountSasPermissions(read=True), + expiry=datetime.utcnow() + timedelta(hours=1), + start=datetime.utcnow() - timedelta(minutes=1), + ) - # Act + account_url = self.account_url(tables_storage_account_name, "table") - sas_table = service.get_table_client(table.table_name) - entities = list(sas_table.list_entities()) + service = self.create_client_from_credential(TableServiceClient, token, endpoint=account_url) - # Assert - assert len(entities) == 2 - assert entities[0]['text'] == u'hello' - assert entities[1]['text'] == u'hello' - # finally: - # self._delete_table(table=table, ts=tsc) - - # for entity in table.list_entities(): - # table.delete_entity(entity["PartitionKey"], entity["RowKey"]) + # Act + + sas_table = service.get_table_client(table.table_name) + entities = list(sas_table.list_entities()) + + # Assert + assert len(entities) == 2 + assert entities[0]['text'] == u'hello' + assert entities[1]['text'] == u'hello' + finally: + self._delete_table(table=table, ts=tsc) class TestTablesUnitTest(TableTestCase): diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index d8e34360c584..e1b67840783f 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -88,6 +88,7 @@ def stop_record_or_playback(test_id, recording_id): headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, verify=False, ) + print("STOPPED: ", result.status_code) elif os.getenv("AZURE_RECORD_MODE") == "playback": result = requests.post( PLAYBACK_STOP_URL, @@ -163,9 +164,17 @@ def transform_args(*args, **kwargs): # this ensures that within this scope, we've monkeypatched the send functionality with patch_requests_func(transform_args): # call the modified function. - value = func(*args, **trimmed_kwargs) - - stop_record_or_playback(test_id, recording_id) + try: + value = func(*args, **trimmed_kwargs) + print("VALUE: {}".format(value)) + except Exception as exc: + # stop_record_or_playback(test_id, recording_id) + # raise + value = None + print(exc) + pass + + stop_record_or_playback(test_id, recording_id) return value From c90a9c6b4c062511612fa54d068e537b5e0043ae Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Tue, 11 May 2021 15:28:38 -0400 Subject: [PATCH 08/90] try/finally --- .../devtools_testutils/proxy_testcase.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index e1b67840783f..59fb2e92fa90 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -88,7 +88,6 @@ def stop_record_or_playback(test_id, recording_id): headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, verify=False, ) - print("STOPPED: ", result.status_code) elif os.getenv("AZURE_RECORD_MODE") == "playback": result = requests.post( PLAYBACK_STOP_URL, @@ -166,15 +165,8 @@ def transform_args(*args, **kwargs): # call the modified function. try: value = func(*args, **trimmed_kwargs) - print("VALUE: {}".format(value)) - except Exception as exc: - # stop_record_or_playback(test_id, recording_id) - # raise - value = None - print(exc) - pass - - stop_record_or_playback(test_id, recording_id) + finally: + stop_record_or_playback(test_id, recording_id) return value From be24617811c691dd567d5c2bd50ef3977d28f452 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Thu, 13 May 2021 15:03:32 -0400 Subject: [PATCH 09/90] splitting proxys into sync and async --- .../azure-data-tables/tests/test_table.py | 1 - .../tests/test_table_async.py | 16 +++++ .../devtools_testutils/aio/__init__.py | 5 ++ .../aio/proxy_testcase_async.py | 60 +++++++++++++++++ .../devtools_testutils/proxy_testcase.py | 66 +++++-------------- 5 files changed, 96 insertions(+), 52 deletions(-) create mode 100644 tools/azure-sdk-tools/devtools_testutils/aio/__init__.py create mode 100644 tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index c58395941496..d52f33b6b60c 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -5,7 +5,6 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from devtools_testutils.proxy_testcase import RecordedByProxyAsync import pytest from datetime import datetime, timedelta diff --git a/sdk/tables/azure-data-tables/tests/test_table_async.py b/sdk/tables/azure-data-tables/tests/test_table_async.py index 9e39f193645f..3b171ebc5cbf 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_async.py @@ -6,6 +6,7 @@ import pytest from devtools_testutils import AzureTestCase +from devtools_testutils.aio import RecordedByProxyAsync from azure.core.credentials import AzureSasCredential, AzureNamedKeyCredential from azure.core.exceptions import ResourceNotFoundError, ResourceExistsError @@ -53,6 +54,7 @@ async def _delete_table(self, ts, table): # --Test cases for tables -------------------------------------------------- @tables_decorator_async + @RecordedByProxyAsync async def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -68,6 +70,7 @@ async def test_create_table(self, tables_storage_account_name, tables_primary_st await ts.delete_table(table_name=table_name) @tables_decorator_async + @RecordedByProxyAsync async def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -87,6 +90,7 @@ async def test_create_table_fail_on_exist(self, tables_storage_account_name, tab await ts.delete_table(table_name=table_name) @tables_decorator_async + @RecordedByProxyAsync async def test_query_tables_per_page(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -116,6 +120,7 @@ async def test_query_tables_per_page(self, tables_storage_account_name, tables_p await ts.delete_table(table_name + str(i)) @tables_decorator_async + @RecordedByProxyAsync async def test_list_tables(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -137,6 +142,7 @@ async def test_list_tables(self, tables_storage_account_name, tables_primary_sto await ts.delete_table(table.table_name) @tables_decorator_async + @RecordedByProxyAsync async def test_query_tables_with_filter(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -158,6 +164,7 @@ async def test_query_tables_with_filter(self, tables_storage_account_name, table await ts.delete_table(table.table_name) @tables_decorator_async + @RecordedByProxyAsync async def test_list_tables_with_num_results(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange prefix = 'listtable' @@ -185,6 +192,7 @@ async def test_list_tables_with_num_results(self, tables_storage_account_name, t assert len(big_page) >= 4 @tables_decorator_async + @RecordedByProxyAsync async def test_list_tables_with_marker(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -218,6 +226,7 @@ async def test_list_tables_with_marker(self, tables_storage_account_name, tables assert tables1 != tables2 @tables_decorator_async + @RecordedByProxyAsync async def test_delete_table_with_existing_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -234,6 +243,7 @@ async def test_delete_table_with_existing_table(self, tables_storage_account_nam assert tables == [] @tables_decorator_async + @RecordedByProxyAsync async def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange @@ -245,6 +255,7 @@ async def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_ # Assert @tables_decorator_async + @RecordedByProxyAsync async def test_get_table_acl(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -262,6 +273,7 @@ async def test_get_table_acl(self, tables_storage_account_name, tables_primary_s await ts.delete_table(table.table_name) @tables_decorator_async + @RecordedByProxyAsync async def test_set_table_acl_with_empty_signed_identifiers(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -280,6 +292,7 @@ async def test_set_table_acl_with_empty_signed_identifiers(self, tables_storage_ await ts.delete_table(table.table_name) @tables_decorator_async + @RecordedByProxyAsync async def test_set_table_acl_with_empty_signed_identifier(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange url = self.account_url(tables_storage_account_name, "table") @@ -301,6 +314,7 @@ async def test_set_table_acl_with_empty_signed_identifier(self, tables_storage_a await ts.delete_table(table.table_name) @tables_decorator_async + @RecordedByProxyAsync async def test_set_table_acl_with_signed_identifiers(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange url = self.account_url(tables_storage_account_name, "table") @@ -325,6 +339,7 @@ async def test_set_table_acl_with_signed_identifiers(self, tables_storage_accoun await ts.delete_table(table.table_name) @tables_decorator_async + @RecordedByProxyAsync async def test_set_table_acl_too_many_ids(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange url = self.account_url(tables_storage_account_name, "table") @@ -344,6 +359,7 @@ async def test_set_table_acl_too_many_ids(self, tables_storage_account_name, tab @pytest.mark.live_test_only @tables_decorator_async + @RecordedByProxyAsync async def test_account_sas(self, tables_storage_account_name, tables_primary_storage_account_key): # SAS URL is calculated from storage key, so this test runs live only diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py b/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py new file mode 100644 index 000000000000..5d6674dc4843 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py @@ -0,0 +1,5 @@ +from .proxy_testcase_async import RecordedByProxyAsync + +__all__ = [ + "RecordedByProxyAsync" +] diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py new file mode 100644 index 000000000000..3b013cadb38a --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -0,0 +1,60 @@ +import functools +from contextlib import contextmanager + +from ..proxy_testcase import ( + get_test_id, + start_record_or_playback, + transform_request, + stop_record_or_playback, +) + +from azure.core.pipeline.transport import AsyncioRequestsTransport + +from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function + + +@contextmanager +def patch_requests_func_async(request_transform): + original_func = AsyncioRequestsTransport.send + + async def combined_call(*args, **kwargs): + adjusted_args, adjusted_kwargs = request_transform(*args, **kwargs) + return await original_func(*adjusted_args, **adjusted_kwargs) + + AsyncioRequestsTransport.send = combined_call + yield None + + AsyncioRequestsTransport.send = original_func + + +def RecordedByProxyAsync(func): + @functools.wraps(func) + async def record_wrap(*args, **kwargs): + test_id = get_test_id() + recording_id = start_record_or_playback(test_id) + + def transform_args(*args, **kwargs): + copied_positional_args = list(args) + request = copied_positional_args[1] + + # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are + # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get + # a recommendation for how to get a valid SSL Cert for localhost + kwargs["connection_verify"] = False + + transform_request(request, recording_id) + + return tuple(copied_positional_args), kwargs + + trimmed_kwargs = {k: v for k, v in kwargs.items()} + trim_kwargs_from_test_function(func, trimmed_kwargs) + + # this ensures that within this scope, we've monkeypatched the send functionality + with patch_requests_func_async(transform_args): + # call the modified function. + value = await func(*args, **trimmed_kwargs) + + stop_record_or_playback(test_id, recording_id) + return value + + return record_wrap diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 59fb2e92fa90..709757acba5b 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -14,7 +14,7 @@ # the functions we patch -from azure.core.pipeline.transport import RequestsTransport, AsyncioRequestsTransport +from azure.core.pipeline.transport import RequestsTransport # the trimming function to clean up incoming arguments to the test function we are wrapping from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function @@ -30,12 +30,13 @@ # unit tests can startup/shutdown the local test proxy # this should also fire the admin mapping updates, and start/end the session for commiting recording updates + @contextmanager def patch_requests_func(request_transform): original_func = RequestsTransport.send def combined_call(*args, **kwargs): - adjusted_args, adjusted_kwargs = request_transform(*args,**kwargs) + adjusted_args, adjusted_kwargs = request_transform(*args, **kwargs) return original_func(*adjusted_args, **adjusted_kwargs) RequestsTransport.send = combined_call @@ -43,33 +44,25 @@ def combined_call(*args, **kwargs): RequestsTransport.send = original_func -@contextmanager -def patch_requests_func_async(request_transform): - original_func = AsyncioRequestsTransport.send - - async def combined_call(*args, **kwargs): - adjusted_args, adjusted_kwargs = request_transform(*args,**kwargs) - return await original_func(*adjusted_args, **adjusted_kwargs) - - AsyncioRequestsTransport.send = combined_call - yield None - - AsyncioRequestsTransport.send = original_func def get_test_id(): # pytest sets the current running test in an environment variable - return os.getenv('PYTEST_CURRENT_TEST').split(" ")[0].replace("::",".") + return os.getenv("PYTEST_CURRENT_TEST").split(" ")[0].replace("::", ".") + def get_current_sha(): result = subprocess.check_output(["git", "rev-parse", "HEAD"]) - #TODO: is this compatible with py27? - return result.decode('utf-8').strip() + # TODO: is this compatible with py27? + return result.decode("utf-8").strip() + def start_record_or_playback(test_id): if os.getenv("AZURE_RECORD_MODE") == "record": result = requests.post( - RECORDING_START_URL, headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha() }, verify=False + RECORDING_START_URL, + headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, + verify=False, ) recording_id = result.headers["x-recording-id"] elif os.getenv("AZURE_RECORD_MODE") == "playback": @@ -81,6 +74,7 @@ def start_record_or_playback(test_id): recording_id = result.headers["x-recording-id"] return recording_id + def stop_record_or_playback(test_id, recording_id): if os.getenv("AZURE_RECORD_MODE") == "record": result = requests.post( @@ -95,6 +89,7 @@ def stop_record_or_playback(test_id, recording_id): verify=False, ) + def transform_request(request, recording_id): upstream_url = request.url.replace("//text", "/text") headers = request.headers @@ -106,37 +101,6 @@ def transform_request(request, recording_id): headers["x-recording-mode"] = os.getenv("AZURE_RECORD_MODE") request.url = PROXY_URL -def RecordedByProxyAsync(func): - @functools.wraps(func) - async def record_wrap(*args, **kwargs): - test_id = get_test_id() - recording_id = get_recording_id(test_id) - - def transform_args(*args, **kwargs): - copied_positional_args = list(args) - request = copied_positional_args[1] - - # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are - # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get - # a recommendation for how to get a valid SSL Cert for localhost - kwargs["connection_verify"] = False - - transform_request(request, recording_id) - - return tuple(copied_positional_args), kwargs - - trimmed_kwargs = {k:v for k,v in kwargs.items()} - trim_kwargs_from_test_function(func, trimmed_kwargs) - - # this ensures that within this scope, we've monkeypatched the send functionality - with patch_requests_func_async(transform_args): - # call the modified function. - value = await func(*args, **trimmed_kwargs) - - stop_record_or_playback(test_id, recording_id) - return value - - return record_wrap def RecordedByProxy(func): @functools.wraps(func) @@ -157,7 +121,7 @@ def transform_args(*args, **kwargs): return tuple(copied_positional_args), kwargs - trimmed_kwargs = {k:v for k,v in kwargs.items()} + trimmed_kwargs = {k: v for k, v in kwargs.items()} trim_kwargs_from_test_function(func, trimmed_kwargs) # this ensures that within this scope, we've monkeypatched the send functionality @@ -170,4 +134,4 @@ def transform_args(*args, **kwargs): return value - return record_wrap \ No newline at end of file + return record_wrap From 7802549b7d9e6b22b61351bc8f66cd32665578ee Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Thu, 13 May 2021 15:48:45 -0400 Subject: [PATCH 10/90] still having issues with async preparer --- sdk/tables/azure-data-tables/tests/test_table_async.py | 5 +++-- .../devtools_testutils/aio/proxy_testcase_async.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/test_table_async.py b/sdk/tables/azure-data-tables/tests/test_table_async.py index 3b171ebc5cbf..dacce1a737fc 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_async.py @@ -5,7 +5,7 @@ import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureTestCase, AzureRecordedTestCase from devtools_testutils.aio import RecordedByProxyAsync from azure.core.credentials import AzureSasCredential, AzureNamedKeyCredential @@ -28,7 +28,7 @@ # ------------------------------------------------------------------------------ -class TableTestAsync(AzureTestCase, AsyncTableTestCase): +class TestStorageTableAsync(AzureRecordedTestCase, AsyncTableTestCase): # --Helpers----------------------------------------------------------------- def _get_table_reference(self, prefix=TEST_TABLE_PREFIX): table_name = self.get_resource_name(prefix) @@ -57,6 +57,7 @@ async def _delete_table(self, ts, table): @RecordedByProxyAsync async def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange + print("running test") account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index 3b013cadb38a..bd537b1e36dd 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -52,7 +52,12 @@ def transform_args(*args, **kwargs): # this ensures that within this scope, we've monkeypatched the send functionality with patch_requests_func_async(transform_args): # call the modified function. - value = await func(*args, **trimmed_kwargs) + print("running") + try: + value = await func(*args, **trimmed_kwargs) + finally: + stop_record_or_playback(test_id, recording_id) + print("stopping") stop_record_or_playback(test_id, recording_id) return value From 80b7c4d2ef33a9d1cd8ac84a8482f0bdb4a8af05 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Thu, 13 May 2021 16:22:50 -0400 Subject: [PATCH 11/90] trying an event loop --- .../azure-data-tables/tests/test_table_async.py | 1 - .../aio/proxy_testcase_async.py | 16 +++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/test_table_async.py b/sdk/tables/azure-data-tables/tests/test_table_async.py index dacce1a737fc..03e7099b2b9a 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_async.py @@ -57,7 +57,6 @@ async def _delete_table(self, ts, table): @RecordedByProxyAsync async def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange - print("running test") account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index bd537b1e36dd..24c88128b7f5 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -1,3 +1,4 @@ +import asyncio import functools from contextlib import contextmanager @@ -27,9 +28,16 @@ async def combined_call(*args, **kwargs): AsyncioRequestsTransport.send = original_func +def run_in_loop(func, *args, **kwargs): + event_loop = asyncio.new_event_loop() + event_loop.run_until_complete( + func(*args, **kwargs) + ) + + def RecordedByProxyAsync(func): @functools.wraps(func) - async def record_wrap(*args, **kwargs): + def record_wrap(*args, **kwargs): test_id = get_test_id() recording_id = start_record_or_playback(test_id) @@ -52,14 +60,12 @@ def transform_args(*args, **kwargs): # this ensures that within this scope, we've monkeypatched the send functionality with patch_requests_func_async(transform_args): # call the modified function. - print("running") try: - value = await func(*args, **trimmed_kwargs) + # value = await func(*args, **trimmed_kwargs) + run_in_loop(func, *args, **kwargs) finally: stop_record_or_playback(test_id, recording_id) - print("stopping") - stop_record_or_playback(test_id, recording_id) return value return record_wrap From fd8d863c538de41c018162bf24c1b4514f10167f Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Thu, 13 May 2021 18:34:31 -0400 Subject: [PATCH 12/90] proxy decorator on the preparer now, simpler code --- .../tests/async_preparers.py | 10 ++++---- .../azure-data-tables/tests/preparers.py | 4 +++- .../azure-data-tables/tests/test_table.py | 23 +------------------ .../tests/test_table_async.py | 18 +-------------- .../aio/proxy_testcase_async.py | 13 ++--------- .../devtools_testutils/proxy_testcase.py | 1 - 6 files changed, 12 insertions(+), 57 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/async_preparers.py b/sdk/tables/azure-data-tables/tests/async_preparers.py index f198df6f88e5..09cd46933261 100644 --- a/sdk/tables/azure-data-tables/tests/async_preparers.py +++ b/sdk/tables/azure-data-tables/tests/async_preparers.py @@ -1,11 +1,13 @@ import functools from azure.core.credentials import AzureNamedKeyCredential +from devtools_testutils.aio import RecordedByProxyAsync from preparers import CosmosPreparer, TablesPreparer, trim_kwargs_from_test_function def cosmos_decorator_async(func, **kwargs): @CosmosPreparer() + @RecordedByProxyAsync def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_cosmos_account_key") name = kwargs.pop("tables_cosmos_account_name") @@ -28,7 +30,8 @@ async def wrapped(*args, **kwargs): def tables_decorator_async(func, **kwargs): @TablesPreparer() - def wrapper(*args, **kwargs): + @RecordedByProxyAsync + async def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_storage_account_key") name = kwargs.pop("tables_storage_account_name") key = AzureNamedKeyCredential(key=key, name=name) @@ -39,9 +42,6 @@ def wrapper(*args, **kwargs): trimmed_kwargs = {k:v for k, v in kwargs.items()} trim_kwargs_from_test_function(func, trimmed_kwargs) - @functools.wraps(func) - async def wrapped(*args, **kwargs): - return await func(*args, **trimmed_kwargs) - return wrapped + return await func(*args, **trimmed_kwargs) return wrapper diff --git a/sdk/tables/azure-data-tables/tests/preparers.py b/sdk/tables/azure-data-tables/tests/preparers.py index baef0629be10..d133ef5a9d98 100644 --- a/sdk/tables/azure-data-tables/tests/preparers.py +++ b/sdk/tables/azure-data-tables/tests/preparers.py @@ -2,7 +2,7 @@ import inspect from azure.core.credentials import AzureNamedKeyCredential -from devtools_testutils import PowerShellPreparer +from devtools_testutils import PowerShellPreparer, RecordedByProxy CosmosPreparer = functools.partial( PowerShellPreparer, "tables", @@ -34,6 +34,7 @@ def trim_kwargs_from_test_function(fn, kwargs): def tables_decorator(func, **kwargs): @TablesPreparer() + @RecordedByProxy def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_storage_account_key") name = kwargs.pop("tables_storage_account_name") @@ -53,6 +54,7 @@ def wrapper(*args, **kwargs): def cosmos_decorator(func, **kwargs): @CosmosPreparer() + @RecordedByProxy def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_cosmos_account_key") name = kwargs.pop("tables_cosmos_account_name") diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index d52f33b6b60c..52485bfd84a8 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -8,10 +8,7 @@ import pytest from datetime import datetime, timedelta -from devtools_testutils import ( - AzureRecordedTestCase, - RecordedByProxy -) +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import ( ResourceTypes, @@ -83,7 +80,6 @@ def _delete_all_tables(self, ts): # --Test cases for tables -------------------------------------------------- @tables_decorator - @RecordedByProxy def test_create_properties(self, tables_storage_account_name, tables_primary_storage_account_key): # # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -107,7 +103,6 @@ def test_create_properties(self, tables_storage_account_name, tables_primary_sto ts.delete_table(table_name) @tables_decorator - @RecordedByProxy def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key): # # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -123,7 +118,6 @@ def test_create_table(self, tables_storage_account_name, tables_primary_storage_ ts.delete_table(table_name) @tables_decorator - @RecordedByProxy def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -143,7 +137,6 @@ def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_pr ts.delete_table(table_name) @tables_decorator - @RecordedByProxy def test_query_tables_per_page(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -176,7 +169,6 @@ def test_query_tables_per_page(self, tables_storage_account_name, tables_primary self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator - @RecordedByProxy def test_create_table_if_exists(self, tables_storage_account_name, tables_primary_storage_account_key): account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) @@ -191,7 +183,6 @@ def test_create_table_if_exists(self, tables_storage_account_name, tables_primar ts.delete_table(table_name) @tables_decorator - @RecordedByProxy def test_create_table_if_exists_new_table(self, tables_storage_account_name, tables_primary_storage_account_key): account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) @@ -204,7 +195,6 @@ def test_create_table_if_exists_new_table(self, tables_storage_account_name, tab ts.delete_table(table_name) @tables_decorator - @RecordedByProxy def test_query_tables(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -227,7 +217,6 @@ def test_query_tables(self, tables_storage_account_name, tables_primary_storage_ self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator - @RecordedByProxy def test_query_tables_with_filter(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -252,7 +241,6 @@ def test_query_tables_with_filter(self, tables_storage_account_name, tables_prim self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator - @RecordedByProxy def test_query_tables_with_num_results(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange prefix = 'listtable' @@ -282,7 +270,6 @@ def test_query_tables_with_num_results(self, tables_storage_account_name, tables self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator - @RecordedByProxy def test_query_tables_with_marker(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -313,7 +300,6 @@ def test_query_tables_with_marker(self, tables_storage_account_name, tables_prim self.sleep(10) # wait for tables to be deleted before proceeding @tables_decorator - @RecordedByProxy def test_delete_table_with_existing_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -329,7 +315,6 @@ def test_delete_table_with_existing_table(self, tables_storage_account_name, tab assert len(existing) == 0 @tables_decorator - @RecordedByProxy def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -338,7 +323,6 @@ def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_storag ts.delete_table(table_name) @tables_decorator - @RecordedByProxy def test_get_table_acl(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange url = self.account_url(tables_storage_account_name, "table") @@ -356,7 +340,6 @@ def test_get_table_acl(self, tables_storage_account_name, tables_primary_storage ts.delete_table(table.table_name) @tables_decorator - @RecordedByProxy def test_set_table_acl_with_empty_signed_identifiers(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -376,7 +359,6 @@ def test_set_table_acl_with_empty_signed_identifiers(self, tables_storage_accoun ts.delete_table(table.table_name) @tables_decorator - @RecordedByProxy def test_set_table_acl_with_empty_signed_identifier(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -399,7 +381,6 @@ def test_set_table_acl_with_empty_signed_identifier(self, tables_storage_account ts.delete_table(table.table_name) @tables_decorator - @RecordedByProxy def test_set_table_acl_with_signed_identifiers(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -425,7 +406,6 @@ def test_set_table_acl_with_signed_identifiers(self, tables_storage_account_name ts.delete_table(table.table_name) @tables_decorator - @RecordedByProxy def test_set_table_acl_too_many_ids(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -447,7 +427,6 @@ def test_set_table_acl_too_many_ids(self, tables_storage_account_name, tables_pr @pytest.mark.live_test_only @tables_decorator - @RecordedByProxy def test_account_sas(self, tables_storage_account_name, tables_primary_storage_account_key): # SAS URL is calculated from storage key, so this test runs live only diff --git a/sdk/tables/azure-data-tables/tests/test_table_async.py b/sdk/tables/azure-data-tables/tests/test_table_async.py index 03e7099b2b9a..bf9ebe58f9c7 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_async.py @@ -6,9 +6,8 @@ import pytest from devtools_testutils import AzureTestCase, AzureRecordedTestCase -from devtools_testutils.aio import RecordedByProxyAsync -from azure.core.credentials import AzureSasCredential, AzureNamedKeyCredential +from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ResourceNotFoundError, ResourceExistsError from azure.data.tables import ( AccessPolicy, @@ -54,7 +53,6 @@ async def _delete_table(self, ts, table): # --Test cases for tables -------------------------------------------------- @tables_decorator_async - @RecordedByProxyAsync async def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -70,7 +68,6 @@ async def test_create_table(self, tables_storage_account_name, tables_primary_st await ts.delete_table(table_name=table_name) @tables_decorator_async - @RecordedByProxyAsync async def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -90,7 +87,6 @@ async def test_create_table_fail_on_exist(self, tables_storage_account_name, tab await ts.delete_table(table_name=table_name) @tables_decorator_async - @RecordedByProxyAsync async def test_query_tables_per_page(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -120,7 +116,6 @@ async def test_query_tables_per_page(self, tables_storage_account_name, tables_p await ts.delete_table(table_name + str(i)) @tables_decorator_async - @RecordedByProxyAsync async def test_list_tables(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -142,7 +137,6 @@ async def test_list_tables(self, tables_storage_account_name, tables_primary_sto await ts.delete_table(table.table_name) @tables_decorator_async - @RecordedByProxyAsync async def test_query_tables_with_filter(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -164,7 +158,6 @@ async def test_query_tables_with_filter(self, tables_storage_account_name, table await ts.delete_table(table.table_name) @tables_decorator_async - @RecordedByProxyAsync async def test_list_tables_with_num_results(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange prefix = 'listtable' @@ -192,7 +185,6 @@ async def test_list_tables_with_num_results(self, tables_storage_account_name, t assert len(big_page) >= 4 @tables_decorator_async - @RecordedByProxyAsync async def test_list_tables_with_marker(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -226,7 +218,6 @@ async def test_list_tables_with_marker(self, tables_storage_account_name, tables assert tables1 != tables2 @tables_decorator_async - @RecordedByProxyAsync async def test_delete_table_with_existing_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -243,7 +234,6 @@ async def test_delete_table_with_existing_table(self, tables_storage_account_nam assert tables == [] @tables_decorator_async - @RecordedByProxyAsync async def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange @@ -255,7 +245,6 @@ async def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_ # Assert @tables_decorator_async - @RecordedByProxyAsync async def test_get_table_acl(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -273,7 +262,6 @@ async def test_get_table_acl(self, tables_storage_account_name, tables_primary_s await ts.delete_table(table.table_name) @tables_decorator_async - @RecordedByProxyAsync async def test_set_table_acl_with_empty_signed_identifiers(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange account_url = self.account_url(tables_storage_account_name, "table") @@ -292,7 +280,6 @@ async def test_set_table_acl_with_empty_signed_identifiers(self, tables_storage_ await ts.delete_table(table.table_name) @tables_decorator_async - @RecordedByProxyAsync async def test_set_table_acl_with_empty_signed_identifier(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange url = self.account_url(tables_storage_account_name, "table") @@ -314,7 +301,6 @@ async def test_set_table_acl_with_empty_signed_identifier(self, tables_storage_a await ts.delete_table(table.table_name) @tables_decorator_async - @RecordedByProxyAsync async def test_set_table_acl_with_signed_identifiers(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange url = self.account_url(tables_storage_account_name, "table") @@ -339,7 +325,6 @@ async def test_set_table_acl_with_signed_identifiers(self, tables_storage_accoun await ts.delete_table(table.table_name) @tables_decorator_async - @RecordedByProxyAsync async def test_set_table_acl_too_many_ids(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange url = self.account_url(tables_storage_account_name, "table") @@ -359,7 +344,6 @@ async def test_set_table_acl_too_many_ids(self, tables_storage_account_name, tab @pytest.mark.live_test_only @tables_decorator_async - @RecordedByProxyAsync async def test_account_sas(self, tables_storage_account_name, tables_primary_storage_account_key): # SAS URL is calculated from storage key, so this test runs live only diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index 24c88128b7f5..4f2ff081784b 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -28,16 +28,8 @@ async def combined_call(*args, **kwargs): AsyncioRequestsTransport.send = original_func -def run_in_loop(func, *args, **kwargs): - event_loop = asyncio.new_event_loop() - event_loop.run_until_complete( - func(*args, **kwargs) - ) - - def RecordedByProxyAsync(func): - @functools.wraps(func) - def record_wrap(*args, **kwargs): + async def record_wrap(*args, **kwargs): test_id = get_test_id() recording_id = start_record_or_playback(test_id) @@ -61,8 +53,7 @@ def transform_args(*args, **kwargs): with patch_requests_func_async(transform_args): # call the modified function. try: - # value = await func(*args, **trimmed_kwargs) - run_in_loop(func, *args, **kwargs) + value = await func(*args, **trimmed_kwargs) finally: stop_record_or_playback(test_id, recording_id) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 709757acba5b..9ce6f462908e 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -103,7 +103,6 @@ def transform_request(request, recording_id): def RecordedByProxy(func): - @functools.wraps(func) def record_wrap(*args, **kwargs): test_id = get_test_id() recording_id = start_record_or_playback(test_id) From 11e3316f219c3e1e16c3aa871ec3e210a154ce77 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 13 May 2021 17:01:34 -0700 Subject: [PATCH 13/90] remove context manager in favor of directly placing monkey patching in record_wrap --- .../devtools_testutils/proxy_testcase.py | 38 +++++++++---------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 709757acba5b..06871490c499 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -31,20 +31,6 @@ # this should also fire the admin mapping updates, and start/end the session for commiting recording updates -@contextmanager -def patch_requests_func(request_transform): - original_func = RequestsTransport.send - - def combined_call(*args, **kwargs): - adjusted_args, adjusted_kwargs = request_transform(*args, **kwargs) - return original_func(*adjusted_args, **adjusted_kwargs) - - RequestsTransport.send = combined_call - yield None - - RequestsTransport.send = original_func - - def get_test_id(): # pytest sets the current running test in an environment variable return os.getenv("PYTEST_CURRENT_TEST").split(" ")[0].replace("::", ".") @@ -124,13 +110,23 @@ def transform_args(*args, **kwargs): trimmed_kwargs = {k: v for k, v in kwargs.items()} trim_kwargs_from_test_function(func, trimmed_kwargs) - # this ensures that within this scope, we've monkeypatched the send functionality - with patch_requests_func(transform_args): - # call the modified function. - try: - value = func(*args, **trimmed_kwargs) - finally: - stop_record_or_playback(test_id, recording_id) + original_transport_func = RequestsTransport.send + # print("Entering patch context. RequestsTransport.send is at {}".format(id(RequestsTransport.send))) + + def combined_call(*args, **kwargs): + adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) + return original_transport_func(*adjusted_args, **adjusted_kwargs) + + RequestsTransport.send = combined_call + # print("Patched context. RequestsTransport.send is now at {}".format(id(RequestsTransport.send))) + + # call the modified function. + try: + value = func(*args, **trimmed_kwargs) + finally: + RequestsTransport.send = original_transport_func + # print("Exiting patch context. RequestsTransport.send is at {}".format(id(RequestsTransport.send))) + stop_record_or_playback(test_id, recording_id) return value From 13cefe0938901465f869551c6f4346e4653712e7 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Fri, 14 May 2021 12:46:37 -0400 Subject: [PATCH 14/90] print statements --- tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 0ea0db87ca6d..780c10cac711 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -92,6 +92,7 @@ def RecordedByProxy(func): def record_wrap(*args, **kwargs): test_id = get_test_id() recording_id = start_record_or_playback(test_id) + print("\n\nRECORDING_ID: ", recording_id) def transform_args(*args, **kwargs): copied_positional_args = list(args) @@ -114,6 +115,10 @@ def transform_args(*args, **kwargs): def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) + req = adjusted_args[1] + print("HEADERS: ", req.headers) + print("BODY: ", req.body) + print("METHOD: ", req.method) return original_transport_func(*adjusted_args, **adjusted_kwargs) RequestsTransport.send = combined_call From a2557fe97097c57725d4653ed74dc3243dafff93 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Fri, 14 May 2021 13:28:12 -0400 Subject: [PATCH 15/90] updates to work for multiple tests --- sdk/tables/azure-data-tables/tests/test_table.py | 5 +++++ .../devtools_testutils/azure_recorded_testcase.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index 52485bfd84a8..936494646288 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -50,6 +50,7 @@ class TestStorageTable(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _get_table_reference(self, prefix=TEST_TABLE_PREFIX): + print("QUAL TEST NAME: ", self.qualified_test_name.encode()) table_name = self.get_resource_name(prefix) return table_name @@ -109,6 +110,7 @@ def test_create_table(self, tables_storage_account_name, tables_primary_storage_ ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) table_name = self._get_table_reference() + print("TABLE NAME: ", table_name) # Act created = ts.create_table(table_name) @@ -123,6 +125,7 @@ def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_pr account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) table_name = self._get_table_reference() + print("TABLE NAME: ", table_name) # Act created = ts.create_table(table_name) @@ -173,6 +176,7 @@ def test_create_table_if_exists(self, tables_storage_account_name, tables_primar account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) table_name = self._get_table_reference() + print("TABLE NAME: ", table_name) t0 = ts.create_table(table_name) t1 = ts.create_table_if_not_exists(table_name) @@ -187,6 +191,7 @@ def test_create_table_if_exists_new_table(self, tables_storage_account_name, tab account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) table_name = self._get_table_reference() + print("TABLE NAME: ", table_name) t = ts.create_table_if_not_exists(table_name) diff --git a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py index 879727490072..5196bb011a8e 100644 --- a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py @@ -177,7 +177,9 @@ def create_basic_client(self, client_class, **kwargs): return self.create_client_from_credential(client_class, credentials, **kwargs) def create_random_name(self, name): - return get_resource_name(name, self.qualified_test_name.encode()) + unique_test_name = os.getenv("PYTEST_CURRENT_TEST").encode("utf-8") + print(unique_test_name) + return get_resource_name(name, unique_test_name) def get_resource_name(self, name): """Alias to create_random_name for back compatibility.""" From d34376a41ed82be3e5b31c4b0ce6c46423c8b51b Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Mon, 17 May 2021 14:16:50 -0400 Subject: [PATCH 16/90] issue with application/xml returned data --- sdk/tables/azure-data-tables/tests/preparers.py | 1 - sdk/tables/azure-data-tables/tests/test_table.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/preparers.py b/sdk/tables/azure-data-tables/tests/preparers.py index d133ef5a9d98..e8561fbb9d6b 100644 --- a/sdk/tables/azure-data-tables/tests/preparers.py +++ b/sdk/tables/azure-data-tables/tests/preparers.py @@ -69,4 +69,3 @@ def wrapper(*args, **kwargs): func(*args, **trimmed_kwargs) return wrapper - diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index 936494646288..ab5892ef70cb 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -52,6 +52,7 @@ class TestStorageTable(AzureRecordedTestCase, TableTestCase): def _get_table_reference(self, prefix=TEST_TABLE_PREFIX): print("QUAL TEST NAME: ", self.qualified_test_name.encode()) table_name = self.get_resource_name(prefix) + print("TABLE_NAME: {}".format(table_name)) return table_name def _create_table(self, ts, prefix=TEST_TABLE_PREFIX, table_list=None): @@ -441,7 +442,6 @@ def test_account_sas(self, tables_storage_account_name, tables_primary_storage_a table = self._create_table(tsc) try: - table = tsc.get_table_client(u"pytablesync669b08d7") entity = { 'PartitionKey': u'test', 'RowKey': u'test1', @@ -465,7 +465,6 @@ def test_account_sas(self, tables_storage_account_name, tables_primary_storage_a service = self.create_client_from_credential(TableServiceClient, token, endpoint=account_url) # Act - sas_table = service.get_table_client(table.table_name) entities = list(sas_table.list_entities()) From 742705d9db101eda8c3caae51f32baa799bed20c Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 18 May 2021 16:28:56 -0700 Subject: [PATCH 17/90] update async decorator to have the same fixes our sync one does --- .../aio/proxy_testcase_async.py | 40 +++++++++---------- .../devtools_testutils/proxy_testcase.py | 2 - 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index 4f2ff081784b..22fe4dd117f2 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -13,21 +13,6 @@ from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function - -@contextmanager -def patch_requests_func_async(request_transform): - original_func = AsyncioRequestsTransport.send - - async def combined_call(*args, **kwargs): - adjusted_args, adjusted_kwargs = request_transform(*args, **kwargs) - return await original_func(*adjusted_args, **adjusted_kwargs) - - AsyncioRequestsTransport.send = combined_call - yield None - - AsyncioRequestsTransport.send = original_func - - def RecordedByProxyAsync(func): async def record_wrap(*args, **kwargs): test_id = get_test_id() @@ -49,13 +34,24 @@ def transform_args(*args, **kwargs): trimmed_kwargs = {k: v for k, v in kwargs.items()} trim_kwargs_from_test_function(func, trimmed_kwargs) - # this ensures that within this scope, we've monkeypatched the send functionality - with patch_requests_func_async(transform_args): - # call the modified function. - try: - value = await func(*args, **trimmed_kwargs) - finally: - stop_record_or_playback(test_id, recording_id) + original_func = AsyncioRequestsTransport.send + + async def combined_call(*args, **kwargs): + adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) + req = adjusted_args[1] + print("HEADERS: ", req.headers) + print("BODY: ", req.body) + print("METHOD: ", req.method) + return await original_func(*adjusted_args, **adjusted_kwargs) + + AsyncioRequestsTransport.send = combined_call + + # call the modified function. + try: + value = await func(*args, **trimmed_kwargs) + finally: + AsyncioRequestsTransport.send = original_func + stop_record_or_playback(test_id, recording_id) return value diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 780c10cac711..e572a5e74829 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -111,7 +111,6 @@ def transform_args(*args, **kwargs): trim_kwargs_from_test_function(func, trimmed_kwargs) original_transport_func = RequestsTransport.send - # print("Entering patch context. RequestsTransport.send is at {}".format(id(RequestsTransport.send))) def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) @@ -122,7 +121,6 @@ def combined_call(*args, **kwargs): return original_transport_func(*adjusted_args, **adjusted_kwargs) RequestsTransport.send = combined_call - # print("Patched context. RequestsTransport.send is now at {}".format(id(RequestsTransport.send))) # call the modified function. try: From bf246795ffea848ce748d364247a9b0fefb90239 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Thu, 20 May 2021 16:02:31 -0400 Subject: [PATCH 18/90] replacing AzureTestCase with AzureRecordedTestCase --- sdk/tables/azure-data-tables/tests/_shared/testcase.py | 2 +- sdk/tables/azure-data-tables/tests/test_retry.py | 4 ++-- sdk/tables/azure-data-tables/tests/test_retry_async.py | 4 ++-- sdk/tables/azure-data-tables/tests/test_table_async.py | 2 +- sdk/tables/azure-data-tables/tests/test_table_batch.py | 4 ++-- .../azure-data-tables/tests/test_table_batch_async.py | 4 ++-- .../azure-data-tables/tests/test_table_batch_cosmos.py | 4 ++-- .../tests/test_table_batch_cosmos_async.py | 4 ++-- sdk/tables/azure-data-tables/tests/test_table_client.py | 4 ++-- .../azure-data-tables/tests/test_table_client_async.py | 6 +++--- .../azure-data-tables/tests/test_table_client_cosmos.py | 4 ++-- .../tests/test_table_client_cosmos_async.py | 6 +++--- sdk/tables/azure-data-tables/tests/test_table_cosmos.py | 4 ++-- .../azure-data-tables/tests/test_table_cosmos_async.py | 4 ++-- sdk/tables/azure-data-tables/tests/test_table_entity.py | 4 ++-- .../azure-data-tables/tests/test_table_entity_async.py | 4 ++-- .../azure-data-tables/tests/test_table_entity_cosmos.py | 4 ++-- .../tests/test_table_entity_cosmos_async.py | 4 ++-- .../tests/test_table_service_properties.py | 4 ++-- .../tests/test_table_service_properties_async.py | 4 ++-- .../tests/test_table_service_properties_cosmos.py | 4 ++-- .../tests/test_table_service_properties_cosmos_async.py | 4 ++-- .../azure-data-tables/tests/test_table_service_stats.py | 4 ++-- .../tests/test_table_service_stats_async.py | 4 ++-- .../tests/test_table_service_stats_cosmos.py | 4 ++-- .../tests/test_table_service_stats_cosmos_async.py | 4 ++-- 26 files changed, 52 insertions(+), 52 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/_shared/testcase.py b/sdk/tables/azure-data-tables/tests/_shared/testcase.py index 19ab9d6279f1..8b7b80e4198c 100644 --- a/sdk/tables/azure-data-tables/tests/_shared/testcase.py +++ b/sdk/tables/azure-data-tables/tests/_shared/testcase.py @@ -13,7 +13,7 @@ import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.credentials import AccessToken, AzureNamedKeyCredential from azure.data.tables import generate_account_sas, AccountSasPermissions, ResourceTypes diff --git a/sdk/tables/azure-data-tables/tests/test_retry.py b/sdk/tables/azure-data-tables/tests/test_retry.py index a0b17ae09979..60a7dcd25cd9 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry.py +++ b/sdk/tables/azure-data-tables/tests/test_retry.py @@ -6,7 +6,7 @@ import unittest import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.exceptions import ( @@ -46,7 +46,7 @@ def send(self, request, **kwargs): return response # --Test Class ----------------------------------------------------------------- -class StorageRetryTest(AzureTestCase, TableTestCase): +class StorageRetryTest(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key, url='table', default_table=True, **kwargs): self.table_name = self.get_resource_name('uttable') diff --git a/sdk/tables/azure-data-tables/tests/test_retry_async.py b/sdk/tables/azure-data-tables/tests/test_retry_async.py index 728d44c1855f..6b057ef9735c 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry_async.py +++ b/sdk/tables/azure-data-tables/tests/test_retry_async.py @@ -6,7 +6,7 @@ import unittest import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.exceptions import ( @@ -45,7 +45,7 @@ async def send(self, request, **kwargs): # --Test Class ----------------------------------------------------------------- -class StorageRetryTest(AzureTestCase, AsyncTableTestCase): +class StorageRetryTest(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key, url='table', default_table=True, **kwargs): self.table_name = self.get_resource_name('uttable') diff --git a/sdk/tables/azure-data-tables/tests/test_table_async.py b/sdk/tables/azure-data-tables/tests/test_table_async.py index bf9ebe58f9c7..ab23ee4f4461 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_async.py @@ -5,7 +5,7 @@ import pytest -from devtools_testutils import AzureTestCase, AzureRecordedTestCase +from devtools_testutils import AzureRecordedTestCase, AzureRecordedTestCase from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ResourceNotFoundError, ResourceExistsError diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch.py b/sdk/tables/azure-data-tables/tests/test_table_batch.py index 6fe8bf82407b..2f3dfafece2b 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch.py @@ -14,7 +14,7 @@ import sys import uuid -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential @@ -45,7 +45,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableBatchTest(AzureTestCase, TableTestCase): +class StorageTableBatchTest(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key): self.ts = TableServiceClient(self.account_url(tables_storage_account_name, "table"), tables_primary_storage_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_async.py b/sdk/tables/azure-data-tables/tests/test_table_batch_async.py index 5be870c48547..13b587de46a2 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_async.py @@ -14,7 +14,7 @@ import sys import uuid -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureSasCredential @@ -44,7 +44,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableBatchTest(AzureTestCase, AsyncTableTestCase): +class StorageTableBatchTest(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key): self.ts = TableServiceClient(self.account_url(tables_storage_account_name, "table"), tables_primary_storage_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py index 0627bd8399a8..54b0bdb4b0ed 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py @@ -13,7 +13,7 @@ import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.exceptions import ( @@ -40,7 +40,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableClientTest(AzureTestCase, TableTestCase): +class StorageTableClientTest(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): self.ts = TableServiceClient(self.account_url(tables_cosmos_account_name, "cosmos"), tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py index a80413e173f4..3cbdac712206 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py @@ -14,7 +14,7 @@ import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.exceptions import ( @@ -42,7 +42,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableBatchTest(AzureTestCase, AsyncTableTestCase): +class StorageTableBatchTest(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): self.ts = TableServiceClient(self.account_url(tables_cosmos_account_name, "cosmos"), tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_client.py b/sdk/tables/azure-data-tables/tests/test_table_client.py index ac0d1df0880b..d9c026cbeda6 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client.py @@ -6,7 +6,7 @@ import pytest import platform -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableServiceClient, TableClient from azure.data.tables import __version__ as VERSION @@ -29,7 +29,7 @@ _CONNECTION_ENDPOINTS_SECONDARY = {'table': 'TableSecondaryEndpoint', 'cosmos': 'TableSecondaryEndpoint'} -class TestTableClient(AzureTestCase, TableTestCase): +class TestTableClient(AzureRecordedTestCase, TableTestCase): @tables_decorator def test_user_agent_custom(self, tables_storage_account_name, tables_primary_storage_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_client_async.py b/sdk/tables/azure-data-tables/tests/test_table_client_async.py index 91360d7d67a0..348165ae7c6d 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client_async.py @@ -7,7 +7,7 @@ import pytest import platform -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.credentials import AzureNamedKeyCredential from azure.data.tables.aio import TableServiceClient, TableClient @@ -15,7 +15,7 @@ from _shared.asynctestcase import AsyncTableTestCase from async_preparers import tables_decorator_async -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase # ------------------------------------------------------------------------------ SERVICES = { TableServiceClient: 'table', @@ -27,7 +27,7 @@ _CONNECTION_ENDPOINTS_SECONDARY = {'table': 'TableSecondaryEndpoint'} -class TestTableClient(AzureTestCase, AsyncTableTestCase): +class TestTableClient(AzureRecordedTestCase, AsyncTableTestCase): @tables_decorator_async async def test_user_agent_default_async(self, tables_storage_account_name, tables_primary_storage_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos.py index 8d506d8a97cf..93a83f9f0f95 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos.py @@ -8,7 +8,7 @@ from time import sleep import sys -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableServiceClient, TableClient from azure.data.tables import __version__ as VERSION @@ -30,7 +30,7 @@ _CONNECTION_ENDPOINTS_SECONDARY = {'table': 'TableSecondaryEndpoint', 'cosmos': 'TableSecondaryEndpoint'} -class TestTableClient(AzureTestCase, TableTestCase): +class TestTableClient(AzureRecordedTestCase, TableTestCase): @pytest.mark.skipif(sys.version_info < (3, 0), reason="Malformed string") @cosmos_decorator diff --git a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py index 8630f231f4d3..743d04771c25 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py @@ -8,7 +8,7 @@ import platform from time import sleep -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables.aio import TableServiceClient, TableClient from azure.data.tables import __version__ as VERSION @@ -16,7 +16,7 @@ from _shared.asynctestcase import AsyncTableTestCase from _shared.testcase import SLEEP_DELAY from async_preparers import cosmos_decorator_async -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase # ------------------------------------------------------------------------------ @@ -29,7 +29,7 @@ _CONNECTION_ENDPOINTS_SECONDARY = {'table': 'TableSecondaryEndpoint', 'cosmos': 'TableSecondaryEndpoint'} -class TestTableClient(AzureTestCase, AsyncTableTestCase): +class TestTableClient(AzureRecordedTestCase, AsyncTableTestCase): @cosmos_decorator_async async def test_user_agent_default_async(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_cosmos.py index 877b00b37700..05a4a7953975 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_cosmos.py @@ -15,7 +15,7 @@ timedelta, ) -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ( @@ -50,7 +50,7 @@ TEST_TABLE_PREFIX = 'pytablesync' # ------------------------------------------------------------------------------ -class StorageTableTest(AzureTestCase, TableTestCase): +class StorageTableTest(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _get_table_reference(self, prefix=TEST_TABLE_PREFIX): diff --git a/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py index 666fa22aa28a..f6a4e5cbcfa5 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py @@ -6,7 +6,7 @@ import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ResourceNotFoundError, ResourceExistsError, HttpResponseError @@ -27,7 +27,7 @@ # ------------------------------------------------------------------------------ -class TableTestAsync(AzureTestCase, AsyncTableTestCase): +class TableTestAsync(AzureRecordedTestCase, AsyncTableTestCase): # --Helpers----------------------------------------------------------------- def _get_table_reference(self, prefix=TEST_TABLE_PREFIX): table_name = self.get_resource_name(prefix) diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity.py b/sdk/tables/azure-data-tables/tests/test_table_entity.py index e3c1441b2bf3..ac6105a9836c 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity.py @@ -14,7 +14,7 @@ from math import isnan import uuid -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import ( TableServiceClient, @@ -40,7 +40,7 @@ # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureTestCase, TableTestCase): +class StorageTableEntityTest(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key, url='table'): self.table_name = self.get_resource_name('uttable') diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_async.py b/sdk/tables/azure-data-tables/tests/test_table_entity_async.py index 204397b3d63e..70357dea7b61 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_async.py @@ -13,7 +13,7 @@ from math import isnan import uuid -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureSasCredential @@ -37,7 +37,7 @@ from _shared.asynctestcase import AsyncTableTestCase from async_preparers import tables_decorator_async -class StorageTableEntityTest(AzureTestCase, AsyncTableTestCase): +class StorageTableEntityTest(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key): account_url = self.account_url(tables_storage_account_name, "table") diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py index 4f57e354e3d9..21efb7606925 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py @@ -15,7 +15,7 @@ from time import sleep import uuid -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.exceptions import ( @@ -40,7 +40,7 @@ # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureTestCase, TableTestCase): +class StorageTableEntityTest(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): self.ts = TableServiceClient(self.account_url(tables_cosmos_account_name, "cosmos"), tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py index 4a539e40be72..cba6dbcf77f5 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py @@ -15,7 +15,7 @@ from time import sleep import uuid -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import ( generate_table_sas, @@ -42,7 +42,7 @@ # TODO: change to `with table_client as client:` to close sessions # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureTestCase, AsyncTableTestCase): +class StorageTableEntityTest(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): account_url = self.account_url(tables_cosmos_account_name, "cosmos") diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties.py index 961b573d39a6..56de42636e22 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties.py @@ -8,7 +8,7 @@ import time import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import ( TableServiceClient, @@ -25,7 +25,7 @@ # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureTestCase, TableTestCase): +class TableServicePropertiesTest(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py index 64a52b976815..aeac37902651 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py @@ -8,7 +8,7 @@ import time import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.exceptions import HttpResponseError @@ -21,7 +21,7 @@ # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureTestCase, TableTestCase): +class TableServicePropertiesTest(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py index 418f7e26d910..58120ead1a34 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py @@ -8,7 +8,7 @@ import time import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.exceptions import HttpResponseError @@ -24,7 +24,7 @@ from preparers import cosmos_decorator # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureTestCase, TableTestCase): +class TableServicePropertiesTest(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py index 13d6db99df0a..f262525662f4 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py @@ -8,7 +8,7 @@ import pytest from time import sleep -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableAnalyticsLogging, Metrics, RetentionPolicy, CorsRule from azure.data.tables.aio import TableServiceClient @@ -19,7 +19,7 @@ from async_preparers import cosmos_decorator_async # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureTestCase, AsyncTableTestCase): +class TableServicePropertiesTest(AzureRecordedTestCase, AsyncTableTestCase): # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py index 9995c4627a1c..e6ad5a818af8 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableServiceClient from _shared.testcase import TableTestCase @@ -20,7 +20,7 @@ '> ' # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureTestCase, TableTestCase): +class TableServiceStatsTest(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_stats_default(self, stats): assert stats is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py index 3e794c8c82e5..f42aa79d0af8 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables.aio import TableServiceClient @@ -22,7 +22,7 @@ # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureTestCase, AsyncTableTestCase): +class TableServiceStatsTest(AzureRecordedTestCase, AsyncTableTestCase): # --Helpers----------------------------------------------------------------- def _assert_stats_default(self, stats): assert stats is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py index 0f54f81704d4..13a4107126e1 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableServiceClient from _shared.testcase import TableTestCase, SLEEP_DELAY @@ -21,7 +21,7 @@ # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureTestCase, TableTestCase): +class TableServiceStatsTest(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_stats_default(self, stats): assert stats is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos_async.py index ff6d4c7ae4d6..6d5d3e393e38 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos_async.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables.aio import TableServiceClient @@ -22,7 +22,7 @@ '> ' # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureTestCase, AsyncTableTestCase): +class TableServiceStatsTest(AzureRecordedTestCase, AsyncTableTestCase): # --Helpers----------------------------------------------------------------- def _assert_stats_default(self, stats): assert stats is not None From 5d52e724b4cd03899200fa7affac8dab29f43511 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Thu, 20 May 2021 16:12:29 -0400 Subject: [PATCH 19/90] renaming test classes --- sdk/tables/azure-data-tables/tests/async_preparers.py | 2 +- sdk/tables/azure-data-tables/tests/test_retry.py | 2 +- sdk/tables/azure-data-tables/tests/test_retry_async.py | 2 +- sdk/tables/azure-data-tables/tests/test_table.py | 2 +- sdk/tables/azure-data-tables/tests/test_table_async.py | 2 +- sdk/tables/azure-data-tables/tests/test_table_batch.py | 2 +- sdk/tables/azure-data-tables/tests/test_table_batch_async.py | 2 +- sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py | 2 +- .../azure-data-tables/tests/test_table_batch_cosmos_async.py | 2 +- sdk/tables/azure-data-tables/tests/test_table_cosmos.py | 4 ++-- sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py | 2 +- sdk/tables/azure-data-tables/tests/test_table_entity.py | 2 +- sdk/tables/azure-data-tables/tests/test_table_entity_async.py | 2 +- .../azure-data-tables/tests/test_table_entity_cosmos.py | 2 +- .../azure-data-tables/tests/test_table_entity_cosmos_async.py | 2 +- .../azure-data-tables/tests/test_table_service_properties.py | 4 ++-- .../tests/test_table_service_properties_async.py | 4 ++-- .../tests/test_table_service_properties_cosmos.py | 4 ++-- .../tests/test_table_service_properties_cosmos_async.py | 4 ++-- .../azure-data-tables/tests/test_table_service_stats.py | 2 +- .../azure-data-tables/tests/test_table_service_stats_async.py | 2 +- .../tests/test_table_service_stats_cosmos.py | 2 +- .../tests/test_table_service_stats_cosmos_async.py | 2 +- 23 files changed, 28 insertions(+), 28 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/async_preparers.py b/sdk/tables/azure-data-tables/tests/async_preparers.py index 09cd46933261..a88daaae9740 100644 --- a/sdk/tables/azure-data-tables/tests/async_preparers.py +++ b/sdk/tables/azure-data-tables/tests/async_preparers.py @@ -8,7 +8,7 @@ def cosmos_decorator_async(func, **kwargs): @CosmosPreparer() @RecordedByProxyAsync - def wrapper(*args, **kwargs): + async def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_cosmos_account_key") name = kwargs.pop("tables_cosmos_account_name") key = AzureNamedKeyCredential(key=key, name=name) diff --git a/sdk/tables/azure-data-tables/tests/test_retry.py b/sdk/tables/azure-data-tables/tests/test_retry.py index 60a7dcd25cd9..6fc007cef8b4 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry.py +++ b/sdk/tables/azure-data-tables/tests/test_retry.py @@ -46,7 +46,7 @@ def send(self, request, **kwargs): return response # --Test Class ----------------------------------------------------------------- -class StorageRetryTest(AzureRecordedTestCase, TableTestCase): +class TestStorageRetry(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key, url='table', default_table=True, **kwargs): self.table_name = self.get_resource_name('uttable') diff --git a/sdk/tables/azure-data-tables/tests/test_retry_async.py b/sdk/tables/azure-data-tables/tests/test_retry_async.py index 6b057ef9735c..0a6dd1b36119 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry_async.py +++ b/sdk/tables/azure-data-tables/tests/test_retry_async.py @@ -45,7 +45,7 @@ async def send(self, request, **kwargs): # --Test Class ----------------------------------------------------------------- -class StorageRetryTest(AzureRecordedTestCase, AsyncTableTestCase): +class TestStorageRetry(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key, url='table', default_table=True, **kwargs): self.table_name = self.get_resource_name('uttable') diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index ab5892ef70cb..eb7345584a77 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -476,7 +476,7 @@ def test_account_sas(self, tables_storage_account_name, tables_primary_storage_a self._delete_table(table=table, ts=tsc) -class TestTablesUnitTest(TableTestCase): +class TestTablesUnit(TableTestCase): tables_storage_account_name = "fake_storage_account" tables_primary_storage_account_key = "fakeXMZjnGsZGvd4bVr3Il5SeHA" credential = AzureNamedKeyCredential(name=tables_storage_account_name, key=tables_primary_storage_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_async.py b/sdk/tables/azure-data-tables/tests/test_table_async.py index ab23ee4f4461..b133ef646c2a 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_async.py @@ -389,7 +389,7 @@ async def test_account_sas(self, tables_storage_account_name, tables_primary_sto await self._delete_table(table=table, ts=tsc) -class TestTablesUnitTest(AsyncTableTestCase): +class TestTablesUnit(AsyncTableTestCase): tables_storage_account_name = "fake_storage_account" tables_primary_storage_account_key = "fakeXMZjnGsZGvd4bVr3Il5SeHA" credential = AzureNamedKeyCredential(name=tables_storage_account_name, key=tables_primary_storage_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch.py b/sdk/tables/azure-data-tables/tests/test_table_batch.py index 2f3dfafece2b..945f32751d0c 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch.py @@ -45,7 +45,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableBatchTest(AzureRecordedTestCase, TableTestCase): +class TestStorageTableBatch(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key): self.ts = TableServiceClient(self.account_url(tables_storage_account_name, "table"), tables_primary_storage_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_async.py b/sdk/tables/azure-data-tables/tests/test_table_batch_async.py index 13b587de46a2..17dd7d11cc8a 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_async.py @@ -44,7 +44,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableBatchTest(AzureRecordedTestCase, AsyncTableTestCase): +class TestStorageTableBatch(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key): self.ts = TableServiceClient(self.account_url(tables_storage_account_name, "table"), tables_primary_storage_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py index 54b0bdb4b0ed..d191f0ad4a91 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py @@ -40,7 +40,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableClientTest(AzureRecordedTestCase, TableTestCase): +class TestStorageTableClient(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): self.ts = TableServiceClient(self.account_url(tables_cosmos_account_name, "cosmos"), tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py index 3cbdac712206..558e180f11d9 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py @@ -42,7 +42,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableBatchTest(AzureRecordedTestCase, AsyncTableTestCase): +class TestStorageTableBatch(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): self.ts = TableServiceClient(self.account_url(tables_cosmos_account_name, "cosmos"), tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_cosmos.py index 05a4a7953975..cec5f575ab87 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_cosmos.py @@ -50,7 +50,7 @@ TEST_TABLE_PREFIX = 'pytablesync' # ------------------------------------------------------------------------------ -class StorageTableTest(AzureRecordedTestCase, TableTestCase): +class TestStorageTable(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _get_table_reference(self, prefix=TEST_TABLE_PREFIX): @@ -275,7 +275,7 @@ def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_cosmos sleep(SLEEP_DELAY) -class TestTableUnitTest(TableTestCase): +class TestTableUnit(TableTestCase): tables_cosmos_account_name = "fake_storage_account" tables_primary_cosmos_account_key = "fakeXMZjnGsZGvd4bVr3Il5SeHA" credential = AzureNamedKeyCredential(name=tables_cosmos_account_name, key=tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py index f6a4e5cbcfa5..daaf9a60f1bb 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py @@ -258,7 +258,7 @@ async def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_ sleep(SLEEP_DELAY) -class TestTableUnitTest(AsyncTableTestCase): +class TestTableUnit(AsyncTableTestCase): tables_cosmos_account_name = "fake_storage_account" tables_primary_cosmos_account_key = "fakeXMZjnGsZGvd4bVr3Il5SeHA" credential = AzureNamedKeyCredential(name=tables_cosmos_account_name, key=tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity.py b/sdk/tables/azure-data-tables/tests/test_table_entity.py index ac6105a9836c..10c6f97d1a1b 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity.py @@ -40,7 +40,7 @@ # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureRecordedTestCase, TableTestCase): +class TestStorageTableEntity(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key, url='table'): self.table_name = self.get_resource_name('uttable') diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_async.py b/sdk/tables/azure-data-tables/tests/test_table_entity_async.py index 70357dea7b61..225393b37be2 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_async.py @@ -37,7 +37,7 @@ from _shared.asynctestcase import AsyncTableTestCase from async_preparers import tables_decorator_async -class StorageTableEntityTest(AzureRecordedTestCase, AsyncTableTestCase): +class TestStorageTableEntity(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key): account_url = self.account_url(tables_storage_account_name, "table") diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py index 21efb7606925..117d8db81de7 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py @@ -40,7 +40,7 @@ # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureRecordedTestCase, TableTestCase): +class TestStorageTableEntity(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): self.ts = TableServiceClient(self.account_url(tables_cosmos_account_name, "cosmos"), tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py index cba6dbcf77f5..e9408c0f28a9 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py @@ -42,7 +42,7 @@ # TODO: change to `with table_client as client:` to close sessions # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureRecordedTestCase, AsyncTableTestCase): +class TestStorageTableEntity(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): account_url = self.account_url(tables_cosmos_account_name, "cosmos") diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties.py index 56de42636e22..846c4ae1193f 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties.py @@ -25,7 +25,7 @@ # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureRecordedTestCase, TableTestCase): +class TableServiceProperties(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None @@ -225,7 +225,7 @@ def test_retention_too_long(self, tables_storage_account_name, tables_primary_st None, None, minute_metrics) -class TestTableUnitTest(TableTestCase): +class TestTableUnit(TableTestCase): def test_retention_no_days(self): # Assert pytest.raises(ValueError, diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py index aeac37902651..d11a58bc9a5d 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py @@ -21,7 +21,7 @@ # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureRecordedTestCase, TableTestCase): +class TableServiceProperties(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None @@ -220,7 +220,7 @@ async def test_retention_too_long_async(self, tables_storage_account_name, table await tsc.set_service_properties(None, None, minute_metrics) -class TestTableUnitTest(TableTestCase): +class TestTableUnit(TableTestCase): @pytest.mark.asyncio async def test_retention_no_days_async(self): diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py index 58120ead1a34..b251ced352fd 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py @@ -24,7 +24,7 @@ from preparers import cosmos_decorator # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureRecordedTestCase, TableTestCase): +class TableServiceProperties(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None @@ -121,7 +121,7 @@ def test_retention_too_long(self, tables_cosmos_account_name, tables_primary_cos self.sleep(SLEEP_DELAY) -class TestTableUnitTest(TableTestCase): +class TestTableUnit(TableTestCase): def test_retention_no_days(self): # Assert diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py index f262525662f4..4c7d2307c5fe 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py @@ -19,7 +19,7 @@ from async_preparers import cosmos_decorator_async # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureRecordedTestCase, AsyncTableTestCase): +class TableServiceProperties(AzureRecordedTestCase, AsyncTableTestCase): # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None @@ -121,7 +121,7 @@ async def test_retention_too_long_async(self, tables_cosmos_account_name, tables self.sleep(SLEEP_DELAY) -class TestTableUnitTest(AsyncTableTestCase): +class TestTableUnit(AsyncTableTestCase): @pytest.mark.asyncio async def test_retention_no_days_async(self): diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py index e6ad5a818af8..f9e7f523dd08 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py @@ -20,7 +20,7 @@ '> ' # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureRecordedTestCase, TableTestCase): +class TableServiceStats(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_stats_default(self, stats): assert stats is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py index f42aa79d0af8..af6a98d8dc68 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py @@ -22,7 +22,7 @@ # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureRecordedTestCase, AsyncTableTestCase): +class TableServiceStats(AzureRecordedTestCase, AsyncTableTestCase): # --Helpers----------------------------------------------------------------- def _assert_stats_default(self, stats): assert stats is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py index 13a4107126e1..1f7daf203aa5 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py @@ -21,7 +21,7 @@ # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureRecordedTestCase, TableTestCase): +class TableServiceStats(AzureRecordedTestCase, TableTestCase): # --Helpers----------------------------------------------------------------- def _assert_stats_default(self, stats): assert stats is not None diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos_async.py index 6d5d3e393e38..8a409a95962c 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos_async.py @@ -22,7 +22,7 @@ '> ' # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureRecordedTestCase, AsyncTableTestCase): +class TableServiceStats(AzureRecordedTestCase, AsyncTableTestCase): # --Helpers----------------------------------------------------------------- def _assert_stats_default(self, stats): assert stats is not None From d4a14ab147ff6fb4abee081b59daeb4dd2a5ce13 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Thu, 20 May 2021 18:43:37 -0400 Subject: [PATCH 20/90] fixing up a few tests, fix for response callbacks --- .../tests/_shared/asynctestcase.py | 7 ++----- sdk/tables/azure-data-tables/tests/test_retry.py | 9 ++++++--- .../azure-data-tables/tests/test_retry_async.py | 10 +++++++--- .../tests/test_table_batch_async.py | 15 +++++---------- .../tests/test_table_batch_cosmos_async.py | 1 - .../devtools_testutils/proxy_testcase.py | 6 ++---- 6 files changed, 22 insertions(+), 26 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/_shared/asynctestcase.py b/sdk/tables/azure-data-tables/tests/_shared/asynctestcase.py index b519d0976731..27d917adab69 100644 --- a/sdk/tables/azure-data-tables/tests/_shared/asynctestcase.py +++ b/sdk/tables/azure-data-tables/tests/_shared/asynctestcase.py @@ -11,11 +11,8 @@ from azure.core.credentials import AccessToken from azure.core.exceptions import ResourceExistsError -from azure.data.tables import ( - EntityProperty, - EdmType, - TableServiceClient, -) +from azure.data.tables import EntityProperty, EdmType +from azure.data.tables.aio import TableServiceClient from devtools_testutils import is_live diff --git a/sdk/tables/azure-data-tables/tests/test_retry.py b/sdk/tables/azure-data-tables/tests/test_retry.py index 93e01a9a6dac..cb0669ec8bfb 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry.py +++ b/sdk/tables/azure-data-tables/tests/test_retry.py @@ -95,7 +95,7 @@ def test_retry_on_timeout(self, tables_storage_account_name, tables_primary_stor default_table=False, retry_mode=RetryMode.Exponential, retry_backoff_factor=1 - ) + ) callback = ResponseCallback(status=200, new_status=408).override_first_status try: @@ -117,7 +117,6 @@ def test_retry_on_socket_timeout(self, tables_storage_account_name, tables_prima retry_mode=RetryMode.Fixed, retry_backoff_factor=1) - new_table_name = self.get_resource_name('uttable') try: with pytest.raises(AzureError) as error: self.ts.get_service_properties() @@ -125,9 +124,13 @@ def test_retry_on_socket_timeout(self, tables_storage_account_name, tables_prima # 3 retries + 1 original == 4 assert retry_transport.count == 4 # This call should succeed on the server side, but fail on the client side due to socket timeout - self.assertTrue('read timeout' in str(error.value), 'Expected socket timeout but got different exception.') + assert 'read timeout' in str(error.value) finally: + self.ts = TableServiceClient( + self.account_url(tables_storage_account_name, "table"), + credential=tables_primary_storage_account_key, + ) self._tear_down() @tables_decorator diff --git a/sdk/tables/azure-data-tables/tests/test_retry_async.py b/sdk/tables/azure-data-tables/tests/test_retry_async.py index 60c091db9bb0..f9a6a2314bac 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry_async.py +++ b/sdk/tables/azure-data-tables/tests/test_retry_async.py @@ -83,7 +83,8 @@ async def test_retry_on_timeout_async(self, tables_storage_account_name, tables_ tables_primary_storage_account_key, default_table=False, retry_mode=RetryMode.Exponential, - retry_backoff_factor=1) + retry_backoff_factor=1 + ) callback = ResponseCallback(status=200, new_status=408).override_first_status @@ -106,7 +107,6 @@ async def test_retry_on_socket_timeout_async(self, tables_storage_account_name, transport=retry_transport, default_table=False) - new_table_name = self.get_resource_name('uttable') try: with pytest.raises(AzureError) as error: await self.ts.get_service_properties() @@ -114,9 +114,13 @@ async def test_retry_on_socket_timeout_async(self, tables_storage_account_name, # 3 retries + 1 original == 4 assert retry_transport.count == 4 # This call should succeed on the server side, but fail on the client side due to socket timeout - self.assertTrue('Timeout on reading' in str(error.value), 'Expected socket timeout but got different exception.') + assert 'Timeout on reading' in str(error.value) finally: + self.ts = TableServiceClient( + self.account_url(tables_storage_account_name, "table"), + credential=tables_primary_storage_account_key, + ) await self._tear_down() @tables_decorator_async diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_async.py b/sdk/tables/azure-data-tables/tests/test_table_batch_async.py index 1ecfc756b394..25081f2f17a6 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_async.py @@ -15,7 +15,7 @@ from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions -from azure.core.credentials import AzureSasCredential +from azure.core.credentials import AzureSasCredential, AzureNamedKeyCredential from azure.core.exceptions import ( ResourceNotFoundError, ClientAuthenticationError @@ -518,14 +518,10 @@ async def test_new_non_existent_table(self, tables_storage_account_name, tables_ @tables_decorator_async async def test_new_invalid_key(self, tables_storage_account_name, tables_primary_storage_account_key): - # Arrange - invalid_key = tables_primary_storage_account_key[0:-6] + "==" # cut off a bit from the end to invalidate - key_list = list(tables_primary_storage_account_key) - - key_list[-6:] = list("0000==") - invalid_key = ''.join(key_list) - - self.ts = TableServiceClient(self.account_url(tables_storage_account_name, "table"), invalid_key) + invalid_key = tables_primary_storage_account_key.named_key.key[0:-6] + "==" # cut off a bit from the end to invalidate + tables_primary_storage_account_key = AzureNamedKeyCredential(tables_storage_account_name, invalid_key) + credential = AzureNamedKeyCredential(name=tables_storage_account_name, key=tables_primary_storage_account_key.named_key.key) + self.ts = TableServiceClient(self.account_url(tables_storage_account_name, "table"), credential) self.table_name = self.get_resource_name('uttable') self.table = self.ts.get_table_client(self.table_name) @@ -557,7 +553,6 @@ async def test_batch_sas_auth(self, tables_storage_account_name, tables_primary_ try: token = self.generate_sas( generate_table_sas, - tables_storage_account_name, tables_primary_storage_account_key, self.table_name, permission=TableSasPermissions(add=True, read=True, update=True, delete=True), diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py index d2552913b166..a11af504f664 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py @@ -565,7 +565,6 @@ async def test_batch_sas_auth(self, tables_cosmos_account_name, tables_primary_c try: token = self.generate_sas( generate_table_sas, - tables_cosmos_account_name, tables_primary_cosmos_account_key, self.table_name, permission=TableSasPermissions(add=True, read=True, update=True, delete=True), diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 780c10cac711..057977031095 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -82,7 +82,8 @@ def transform_request(request, recording_id): # quiet passthrough if neither are set if os.getenv("AZURE_RECORD_MODE") == "record" or os.getenv("AZURE_RECORD_MODE") == "playback": - headers["x-recording-upstream-base-uri"] = upstream_url + if headers.get("x-recording-upstream-base-uri", None) is None: + headers["x-recording-upstream-base-uri"] = upstream_url headers["x-recording-id"] = recording_id headers["x-recording-mode"] = os.getenv("AZURE_RECORD_MODE") request.url = PROXY_URL @@ -116,9 +117,6 @@ def transform_args(*args, **kwargs): def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) req = adjusted_args[1] - print("HEADERS: ", req.headers) - print("BODY: ", req.body) - print("METHOD: ", req.method) return original_transport_func(*adjusted_args, **adjusted_kwargs) RequestsTransport.send = combined_call From 103836f3d6769f2eda092ebced11a32e4b3f2e2f Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Fri, 21 May 2021 10:08:13 -0400 Subject: [PATCH 21/90] small fix --- tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 057977031095..e40c176fc180 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -54,7 +54,8 @@ def start_record_or_playback(test_id): elif os.getenv("AZURE_RECORD_MODE") == "playback": result = requests.post( PLAYBACK_START_URL, - headers={"x-recording-file": test_id, "x-recording-id": recording_id}, + # headers={"x-recording-file": test_id, "x-recording-id": recording_id}, + headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, verify=False, ) recording_id = result.headers["x-recording-id"] From 57b876742f03bdae59afe73ece7455c5f19e3926 Mon Sep 17 00:00:00 2001 From: seankane-msft Date: Fri, 21 May 2021 10:29:17 -0400 Subject: [PATCH 22/90] small changes --- sdk/tables/azure-data-tables/tests/test_table.py | 6 +----- .../azure-data-tables/tests/test_table_client_async.py | 1 - .../tests/test_table_client_cosmos_async.py | 1 - .../azure-data-tables/tests/test_table_cosmos_async.py | 2 +- .../tests/test_table_service_properties.py | 2 +- .../tests/test_table_service_properties_async.py | 2 +- .../tests/test_table_service_properties_cosmos.py | 2 +- .../tests/test_table_service_properties_cosmos_async.py | 2 +- 8 files changed, 6 insertions(+), 12 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index 35a12e468eff..dd8991cca012 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -67,7 +67,6 @@ def test_create_table(self, tables_storage_account_name, tables_primary_storage_ ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) table_name = self._get_table_reference() - print("TABLE NAME: ", table_name) # Act created = ts.create_table(table_name) @@ -82,7 +81,6 @@ def test_create_table_fail_on_exist(self, tables_storage_account_name, tables_pr account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) table_name = self._get_table_reference() - print("TABLE NAME: ", table_name) # Act created = ts.create_table(table_name) @@ -133,7 +131,6 @@ def test_create_table_if_exists(self, tables_storage_account_name, tables_primar account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) table_name = self._get_table_reference() - print("TABLE NAME: ", table_name) t0 = ts.create_table(table_name) t1 = ts.create_table_if_not_exists(table_name) @@ -148,7 +145,6 @@ def test_create_table_if_exists_new_table(self, tables_storage_account_name, tab account_url = self.account_url(tables_storage_account_name, "table") ts = self.create_client_from_credential(TableServiceClient, tables_primary_storage_account_key, endpoint=account_url) table_name = self._get_table_reference() - print("TABLE NAME: ", table_name) t = ts.create_table_if_not_exists(table_name) @@ -429,7 +425,7 @@ def test_account_sas(self, tables_storage_account_name, tables_primary_storage_a tsc.delete_table(table.table_name) -class TestTablesUnit(TableTestCase): +class TestTablesUnitTest(TableTestCase): tables_storage_account_name = "fake_storage_account" tables_primary_storage_account_key = "fakeXMZjnGsZGvd4bVr3Il5SeHA" credential = AzureNamedKeyCredential(name=tables_storage_account_name, key=tables_primary_storage_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_client_async.py b/sdk/tables/azure-data-tables/tests/test_table_client_async.py index 348165ae7c6d..eeb0c9a39fb7 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client_async.py @@ -15,7 +15,6 @@ from _shared.asynctestcase import AsyncTableTestCase from async_preparers import tables_decorator_async -from devtools_testutils import AzureRecordedTestCase # ------------------------------------------------------------------------------ SERVICES = { TableServiceClient: 'table', diff --git a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py index 743d04771c25..05cfbbe9baff 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py @@ -16,7 +16,6 @@ from _shared.asynctestcase import AsyncTableTestCase from _shared.testcase import SLEEP_DELAY from async_preparers import cosmos_decorator_async -from devtools_testutils import AzureRecordedTestCase # ------------------------------------------------------------------------------ diff --git a/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py index e09c06377b90..47c4430c4555 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py @@ -215,7 +215,7 @@ async def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_ sleep(SLEEP_DELAY) -class TestTableUnit(AsyncTableTestCase): +class TestTableUnitTest(AsyncTableTestCase): tables_cosmos_account_name = "fake_storage_account" tables_primary_cosmos_account_key = "fakeXMZjnGsZGvd4bVr3Il5SeHA" credential = AzureNamedKeyCredential(name=tables_cosmos_account_name, key=tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties.py index 4c08f91820c1..14a1e6e1ebd7 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties.py @@ -150,7 +150,7 @@ def test_retention_too_long(self, tables_storage_account_name, tables_primary_st None, None, minute_metrics) -class TestTableUnit(TableTestCase): +class TestTableUnitTest(TableTestCase): def test_retention_no_days(self): # Assert pytest.raises(ValueError, diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py index 82fe98bd254e..2ff8eb89998c 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py @@ -145,7 +145,7 @@ async def test_retention_too_long_async(self, tables_storage_account_name, table await tsc.set_service_properties(None, None, minute_metrics) -class TestTableUnit(TableTestCase): +class TestTableUnitTest(TableTestCase): @pytest.mark.asyncio async def test_retention_no_days_async(self): diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py index 3b49ec86d6ef..69ee7f64e969 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py @@ -42,7 +42,7 @@ def test_retention_too_long(self, tables_cosmos_account_name, tables_primary_cos tsc.set_service_properties(None, None, minute_metrics) -class TestTableUnit(TableTestCase): +class TestTableUnitTest(TableTestCase): def test_retention_no_days(self): # Assert diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py index 25536f889c49..f1d61db01cd3 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py @@ -43,7 +43,7 @@ async def test_retention_too_long_async(self, tables_cosmos_account_name, tables await tsc.set_service_properties(None, None, minute_metrics) -class TestTableUnit(AsyncTableTestCase): +class TestTableUnitTest(AsyncTableTestCase): @pytest.mark.asyncio async def test_retention_no_days_async(self): From 8eb589485d66296f5948d2151635fff2a88684e8 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 15 Jun 2021 14:03:03 -0700 Subject: [PATCH 23/90] resolve failing auth by updating how we treat x-base-upstream-uri --- .../devtools_testutils/proxy_testcase.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 32411d3a438a..c14b42304565 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -8,8 +8,15 @@ import pdb import os import requests -from contextlib import contextmanager +try: + # py3 + import urllib.parse as url_parse +except: + # py2 + import urlparse as url_parse + +from contextlib import contextmanager import subprocess @@ -20,7 +27,7 @@ from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function # defaults -PROXY_URL = "http://localhost:5001" +PROXY_URL = "http://localhost:5000" RECORDING_START_URL = "{}/record/start".format(PROXY_URL) RECORDING_STOP_URL = "{}/record/stop".format(PROXY_URL) PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL) @@ -76,6 +83,9 @@ def stop_record_or_playback(test_id, recording_id): verify=False, ) +def get_proxy_netloc(): + parsed_result = url_parse.urlparse(PROXY_URL) + return { "scheme": parsed_result.scheme, "netloc": parsed_result.netloc } def transform_request(request, recording_id): upstream_url = request.url.replace("//text", "/text") @@ -83,11 +93,13 @@ def transform_request(request, recording_id): # quiet passthrough if neither are set if os.getenv("AZURE_RECORD_MODE") == "record" or os.getenv("AZURE_RECORD_MODE") == "playback": + parsed_result = url_parse.urlparse(request.url) + updated_target = parsed_result._replace(**get_proxy_netloc()).geturl() if headers.get("x-recording-upstream-base-uri", None) is None: - headers["x-recording-upstream-base-uri"] = upstream_url + headers["x-recording-upstream-base-uri"] = "{}://{}".format(parsed_result.scheme, parsed_result.netloc) headers["x-recording-id"] = recording_id headers["x-recording-mode"] = os.getenv("AZURE_RECORD_MODE") - request.url = PROXY_URL + request.url = updated_target def RecordedByProxy(func): From 637a351beaa439db2455d7a018c9370f6630c1ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Tue, 20 Jul 2021 14:55:27 -0700 Subject: [PATCH 24/90] Successful recording --- ...bleServiceStats.test_table_service_stats_f | 35 +++++++++++++++++++ .../scenario_tests/preparers.py | 4 ++- .../devtools_testutils/proxy_testcase.py | 12 +++---- 3 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f diff --git a/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f b/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f new file mode 100644 index 000000000000..298613f75eb1 --- /dev/null +++ b/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f @@ -0,0 +1,35 @@ +{ + "Entries": [ + { + "RequestUri": "https://mcpatinostorage-secondary.table.core.windows.net/?restype=service\u0026comp=stats", + "RequestMethod": "GET", + "RequestHeaders": { + "Accept": "application/xml", + "Accept-Encoding": "gzip, deflate", + "Authorization": "Sanitized", + "Connection": "keep-alive", + "Date": "Tue, 20 Jul 2021 21:38:24 GMT", + "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.0 (Windows-10-10.0.19041-SP0)", + "x-ms-client-request-id": "cf83178a-e9a2-11eb-9ee0-c8348e51ab51", + "x-ms-date": "Tue, 20 Jul 2021 21:38:24 GMT", + "x-ms-version": "2019-02-02" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Content-Type": "application/xml", + "Date": "Tue, 20 Jul 2021 21:38:23 GMT", + "Server": [ + "Windows-Azure-Table/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "Transfer-Encoding": "chunked", + "x-ms-client-request-id": "cf83178a-e9a2-11eb-9ee0-c8348e51ab51", + "x-ms-request-id": "56a3083e-1002-0069-5faf-7d8b05000000", + "x-ms-version": "2019-02-02" + }, + "ResponseBody": "\uFEFF\u003C?xml version=\u00221.0\u0022 encoding=\u0022utf-8\u0022?\u003E\u003CStorageServiceStats\u003E\u003CGeoReplication\u003E\u003CStatus\u003Elive\u003C/Status\u003E\u003CLastSyncTime\u003ETue, 20 Jul 2021 21:34:44 GMT\u003C/LastSyncTime\u003E\u003C/GeoReplication\u003E\u003C/StorageServiceStats\u003E" + } + ], + "Variables": {} +} \ No newline at end of file diff --git a/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py b/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py index 1bc992962882..73f1150db866 100644 --- a/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py +++ b/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py @@ -135,7 +135,9 @@ def _preparer_wrapper(test_class_instance, **kwargs): ) if test_class_instance.is_live: - test_class_instance.scrubber.register_name_pair(resource_name, self.moniker) + # Adding this for new proxy testcase + if hasattr(test_class_instance, "scrubber"): + test_class_instance.scrubber.register_name_pair(resource_name, self.moniker) # We shouldn't trim the same kwargs that we use for deletion, # we may remove some of the variables we needed to do the delete. diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index c14b42304565..636e683d0f32 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -4,8 +4,6 @@ # license information. # -------------------------------------------------------------------------- -import functools -import pdb import os import requests @@ -16,10 +14,8 @@ # py2 import urlparse as url_parse -from contextlib import contextmanager import subprocess - # the functions we patch from azure.core.pipeline.transport import RequestsTransport @@ -33,8 +29,8 @@ PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL) PLAYBACK_STOP_URL = "{}/playback/stop".format(PROXY_URL) -# TODO, create a pytest scope="session" implementation that can be added to a fixture such that that can -# unit tests can startup/shutdown the local test proxy +# TODO, create a pytest scope="session" implementation that can be added to a fixture such that unit tests can +# startup/shutdown the local test proxy # this should also fire the admin mapping updates, and start/end the session for commiting recording updates @@ -83,9 +79,11 @@ def stop_record_or_playback(test_id, recording_id): verify=False, ) + def get_proxy_netloc(): parsed_result = url_parse.urlparse(PROXY_URL) - return { "scheme": parsed_result.scheme, "netloc": parsed_result.netloc } + return {"scheme": parsed_result.scheme, "netloc": parsed_result.netloc} + def transform_request(request, recording_id): upstream_url = request.url.replace("//text", "/text") From 53fd4035d4fc2e62f2d9bff76cad3fab8857de67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Wed, 21 Jul 2021 15:22:53 -0700 Subject: [PATCH 25/90] e2e testing w/ manual sanitizer setup --- sdk/tables/azure-data-tables/tests/preparers.py | 2 +- ...tTableServiceStats.test_table_service_stats_f | 16 ++++++++-------- .../azure_recorded_testcase.py | 1 - .../devtools_testutils/proxy_testcase.py | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/preparers.py b/sdk/tables/azure-data-tables/tests/preparers.py index 13778286c3f7..3cb81ecbc336 100644 --- a/sdk/tables/azure-data-tables/tests/preparers.py +++ b/sdk/tables/azure-data-tables/tests/preparers.py @@ -16,7 +16,7 @@ TablesPreparer = functools.partial( PowerShellPreparer, "tables", - tables_storage_account_name="fake_table_account", + tables_storage_account_name="faketableaccount", tables_primary_storage_account_key="faketablesaccountkey", ) diff --git a/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f b/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f index 298613f75eb1..1a1a0b641a94 100644 --- a/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f +++ b/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f @@ -1,34 +1,34 @@ { "Entries": [ { - "RequestUri": "https://mcpatinostorage-secondary.table.core.windows.net/?restype=service\u0026comp=stats", + "RequestUri": "https://faketableaccount-secondary.table.core.windows.net/?restype=service\u0026comp=stats", "RequestMethod": "GET", "RequestHeaders": { "Accept": "application/xml", "Accept-Encoding": "gzip, deflate", "Authorization": "Sanitized", "Connection": "keep-alive", - "Date": "Tue, 20 Jul 2021 21:38:24 GMT", + "Date": "Wed, 21 Jul 2021 22:10:01 GMT", "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.0 (Windows-10-10.0.19041-SP0)", - "x-ms-client-request-id": "cf83178a-e9a2-11eb-9ee0-c8348e51ab51", - "x-ms-date": "Tue, 20 Jul 2021 21:38:24 GMT", + "x-ms-client-request-id": "652f28dd-ea70-11eb-8b90-c8348e51ab51", + "x-ms-date": "Wed, 21 Jul 2021 22:10:01 GMT", "x-ms-version": "2019-02-02" }, "RequestBody": null, "StatusCode": 200, "ResponseHeaders": { "Content-Type": "application/xml", - "Date": "Tue, 20 Jul 2021 21:38:23 GMT", + "Date": "Wed, 21 Jul 2021 22:10:01 GMT", "Server": [ "Windows-Azure-Table/1.0", "Microsoft-HTTPAPI/2.0" ], "Transfer-Encoding": "chunked", - "x-ms-client-request-id": "cf83178a-e9a2-11eb-9ee0-c8348e51ab51", - "x-ms-request-id": "56a3083e-1002-0069-5faf-7d8b05000000", + "x-ms-client-request-id": "652f28dd-ea70-11eb-8b90-c8348e51ab51", + "x-ms-request-id": "453e627a-3002-0023-147d-7e288a000000", "x-ms-version": "2019-02-02" }, - "ResponseBody": "\uFEFF\u003C?xml version=\u00221.0\u0022 encoding=\u0022utf-8\u0022?\u003E\u003CStorageServiceStats\u003E\u003CGeoReplication\u003E\u003CStatus\u003Elive\u003C/Status\u003E\u003CLastSyncTime\u003ETue, 20 Jul 2021 21:34:44 GMT\u003C/LastSyncTime\u003E\u003C/GeoReplication\u003E\u003C/StorageServiceStats\u003E" + "ResponseBody": "\uFEFF\u003C?xml version=\u00221.0\u0022 encoding=\u0022utf-8\u0022?\u003E\u003CStorageServiceStats\u003E\u003CGeoReplication\u003E\u003CStatus\u003Elive\u003C/Status\u003E\u003CLastSyncTime\u003EWed, 21 Jul 2021 22:06:46 GMT\u003C/LastSyncTime\u003E\u003C/GeoReplication\u003E\u003C/StorageServiceStats\u003E" } ], "Variables": {} diff --git a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py index 5196bb011a8e..e15321dadefe 100644 --- a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py @@ -27,7 +27,6 @@ load_dotenv(find_dotenv()) -print(os.environ["AZURE_RECORD_MODE"]) class AzureRecordedTestCase(object): diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 636e683d0f32..c01048995bc4 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -86,6 +86,7 @@ def get_proxy_netloc(): def transform_request(request, recording_id): + """Redirect the request to the test proxy, and store the original request URI in a header""" upstream_url = request.url.replace("//text", "/text") headers = request.headers @@ -104,7 +105,6 @@ def RecordedByProxy(func): def record_wrap(*args, **kwargs): test_id = get_test_id() recording_id = start_record_or_playback(test_id) - print("\n\nRECORDING_ID: ", recording_id) def transform_args(*args, **kwargs): copied_positional_args = list(args) From 56c2688f480abc46bc72fe3e0acd4ad8e02dff79 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 16:24:19 -0700 Subject: [PATCH 26/90] add initial version of job that will run docker --- eng/pipelines/test-with-proxy.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 eng/pipelines/test-with-proxy.yml diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml new file mode 100644 index 000000000000..4103bd4cf5d8 --- /dev/null +++ b/eng/pipelines/test-with-proxy.yml @@ -0,0 +1,22 @@ +trigger: none + +variables: + PythonVersion: '3.9' + +jobs: + - job: 'rpat' + displayName: 'Run Proxy and Test' + + pool: + vmImage: 'ubuntu-20.04' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + + - pwsh: | + # if online, we expect this to show up. + Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" + displayName: 'Check Proxy Availability' From 2dda828f7717dea72b1430f4468151a9c00f3859 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 16:36:30 -0700 Subject: [PATCH 27/90] run docker container --- eng/pipelines/test-with-proxy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 4103bd4cf5d8..678a384f346c 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -16,6 +16,10 @@ jobs: inputs: versionSpec: $(PythonVersion) + - pwsh: | + docker run -v $(Build.SourcesDirectory):/etc/testproxy -p 5001:5001 -p 5000:5000 azsdkengsys.azurecr.io/engsys/ubuntu_testproxy_server:latest + displayName: 'Run the docker container' + - pwsh: | # if online, we expect this to show up. Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" From de8c463c0ef7fe39decd6d3d2edf62b976ac8628 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 16:38:55 -0700 Subject: [PATCH 28/90] ensure that the run starts in the background --- eng/pipelines/test-with-proxy.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 678a384f346c..efc5235bba72 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -17,7 +17,12 @@ jobs: versionSpec: $(PythonVersion) - pwsh: | - docker run -v $(Build.SourcesDirectory):/etc/testproxy -p 5001:5001 -p 5000:5000 azsdkengsys.azurecr.io/engsys/ubuntu_testproxy_server:latest + docker run ` + --detach ` + -v $(Build.SourcesDirectory):/etc/testproxy ` + -p 5001:5001 ` + -p 5000:5000 ` + azsdkengsys.azurecr.io/engsys/ubuntu_testproxy_server:latest displayName: 'Run the docker container' - pwsh: | From 6f1cdb8199768d81b6b110ad391380af54105c6c Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 16:41:43 -0700 Subject: [PATCH 29/90] add another webrequest invocation --- eng/pipelines/test-with-proxy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index efc5235bba72..25f144063606 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -23,6 +23,8 @@ jobs: -p 5001:5001 ` -p 5000:5000 ` azsdkengsys.azurecr.io/engsys/ubuntu_testproxy_server:latest + + Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" displayName: 'Run the docker container' - pwsh: | From 4e031b4a9c3b4b28e3733ba57ec4b63319b7a826 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 16:47:23 -0700 Subject: [PATCH 30/90] additional debug outputs --- eng/pipelines/test-with-proxy.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 25f144063606..eb4f45aa6b81 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -23,11 +23,29 @@ jobs: -p 5001:5001 ` -p 5000:5000 ` azsdkengsys.azurecr.io/engsys/ubuntu_testproxy_server:latest - - Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" displayName: 'Run the docker container' - pwsh: | - # if online, we expect this to show up. + docker container ls -a + displayName: "View running docker containers" + continueOnError: true + + - pwsh: | Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" - displayName: 'Check Proxy Availability' + displayName: 'http://localhost:5000/Info/Available' + continueOnError: true + + - pwsh: | + Invoke-WebRequest -Uri "https://localhost:5000/Info/Available" + displayName: 'https://localhost:5000/Info/Available' + continueOnError: true + + - pwsh: | + Invoke-WebRequest -Uri "http://localhost:5001/Info/Available" + displayName: 'http://localhost:5001/Info/Available' + continueOnError: true + + - pwsh: | + Invoke-WebRequest -Uri "https://localhost:5001/Info/Available" + displayName: 'https://localhost:5001/Info/Available' + continueOnError: true \ No newline at end of file From 81917b9fb83a773aa4c2996a4db654388c75a0ef Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 17:17:54 -0700 Subject: [PATCH 31/90] deliberately break the docker call by swapping to windows --- eng/pipelines/test-with-proxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index eb4f45aa6b81..486d1cd154c5 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -8,7 +8,7 @@ jobs: displayName: 'Run Proxy and Test' pool: - vmImage: 'ubuntu-20.04' + vmImage: 'windows-2019' steps: - task: UsePythonVersion@0 From 647df7159b238e49b18845428aa37b1eccc8d8b2 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 17:24:43 -0700 Subject: [PATCH 32/90] more debugging options --- eng/pipelines/test-with-proxy.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 486d1cd154c5..3e8693ae83ed 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -16,6 +16,15 @@ jobs: inputs: versionSpec: $(PythonVersion) + - pwsh: | + choco --version + displayName: "Is Choco available?" + continueOnError: true + + - pwsh: | + Get-ChildItem -Recurse C:/ + displayName: "Dump the contents of disk" + - pwsh: | docker run ` --detach ` From 6cbf6e02ccb49a3fa6fac978a73b9a7fd6a07ddd Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 17:42:47 -0700 Subject: [PATCH 33/90] add environment dump. add attempt to detect dockercli --- eng/pipelines/test-with-proxy.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 3e8693ae83ed..3fb1eb3cd7b9 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -22,8 +22,14 @@ jobs: continueOnError: true - pwsh: | - Get-ChildItem -Recurse C:/ - displayName: "Dump the contents of disk" + dir env: + displayName: "Dump Env" + + - pwsh: | + cd "C:\Program Files\Docker\Docker" + ls + displayName: 'Ensure appropriate image type' + continueOnError: true - pwsh: | docker run ` From dacf7e52d9e0fe4c4b845f9432f31504ae81d6ee Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 17:47:06 -0700 Subject: [PATCH 34/90] output path --- eng/pipelines/test-with-proxy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 3fb1eb3cd7b9..f59e5d73904b 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -26,8 +26,7 @@ jobs: displayName: "Dump Env" - pwsh: | - cd "C:\Program Files\Docker\Docker" - ls + Write-Host "$env:PATH" displayName: 'Ensure appropriate image type' continueOnError: true From 1a17f13637a50e719a1bb462390b050351fae781 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 17:50:52 -0700 Subject: [PATCH 35/90] dump the docker version --- eng/pipelines/test-with-proxy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index f59e5d73904b..b30bfb8b1bc1 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -26,8 +26,8 @@ jobs: displayName: "Dump Env" - pwsh: | - Write-Host "$env:PATH" - displayName: 'Ensure appropriate image type' + Get-ChildItem -Recurse C:\Program Files\Docker + displayName: 'Dump Details of Docker Install' continueOnError: true - pwsh: | From 9e3e39446f2e019cc3bd5937d3eab1019cbdcbea Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 17:54:33 -0700 Subject: [PATCH 36/90] need to surround in quotes --- eng/pipelines/test-with-proxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index b30bfb8b1bc1..cb3e67022085 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -26,7 +26,7 @@ jobs: displayName: "Dump Env" - pwsh: | - Get-ChildItem -Recurse C:\Program Files\Docker + Get-ChildItem -Recurse "C:/Program Files/Docker" displayName: 'Dump Details of Docker Install' continueOnError: true From 1875f063c8bef552ec068a56e8057113f7e2c97a Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 18:06:21 -0700 Subject: [PATCH 37/90] awesome --- eng/pipelines/test-with-proxy.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index cb3e67022085..7bcc499f551a 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -30,6 +30,14 @@ jobs: displayName: 'Dump Details of Docker Install' continueOnError: true + - pwsh: | + choco install docker-cli + displayName: 'Install docker-cli only' + + - pwsh: | + dockercli -SwitchDaemon + displayName: 'Install docker-cli only' + - pwsh: | docker run ` --detach ` From 619e1a9c97977c13afec691c6e8bc4704f18f03c Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 18:19:44 -0700 Subject: [PATCH 38/90] invoke switchdaemon --- eng/pipelines/test-with-proxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 7bcc499f551a..865b8cf0ba96 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -35,7 +35,7 @@ jobs: displayName: 'Install docker-cli only' - pwsh: | - dockercli -SwitchDaemon + C:\ProgramData\chocolatey\lib\docker-cli\tools\docker.exe -SwitchDaemon displayName: 'Install docker-cli only' - pwsh: | From 906980aed02a126a2b739be90508c4927b9d5ede Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 18:30:07 -0700 Subject: [PATCH 39/90] another attempt to get access to dockercli.exe --- eng/pipelines/test-with-proxy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 865b8cf0ba96..d6e31a61445a 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -31,11 +31,11 @@ jobs: continueOnError: true - pwsh: | - choco install docker-cli + choco install docker-desktop displayName: 'Install docker-cli only' - pwsh: | - C:\ProgramData\chocolatey\lib\docker-cli\tools\docker.exe -SwitchDaemon + C:\ProgramData\chocolatey\lib\docker-desktop\tools\dockercli.exe -SwitchDaemon displayName: 'Install docker-cli only' - pwsh: | From 5404ea757856d488b1c4a86ffe29b533089255be Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 18:42:05 -0700 Subject: [PATCH 40/90] attempt running from the installed docker --- eng/pipelines/test-with-proxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index d6e31a61445a..8ba7629922ad 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -35,7 +35,7 @@ jobs: displayName: 'Install docker-cli only' - pwsh: | - C:\ProgramData\chocolatey\lib\docker-desktop\tools\dockercli.exe -SwitchDaemon + Get-ChildItem -Recurse "C:\Program Files\Docker\Docker" displayName: 'Install docker-cli only' - pwsh: | From 1da293e80686af3876af567fd21eadfbfdd2c4c5 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 18:50:52 -0700 Subject: [PATCH 41/90] get dockercli.exe --- eng/pipelines/test-with-proxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 8ba7629922ad..cbd4142452de 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -35,7 +35,7 @@ jobs: displayName: 'Install docker-cli only' - pwsh: | - Get-ChildItem -Recurse "C:\Program Files\Docker\Docker" + C:\Program Files\Docker\Docker\DockerCli.exe -SwitchLinuxEngine displayName: 'Install docker-cli only' - pwsh: | From 37848cd326a8eef75d5a0310d495001e493c520e Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 18:51:11 -0700 Subject: [PATCH 42/90] need to surround with quotes --- eng/pipelines/test-with-proxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index cbd4142452de..b0be0dd1a950 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -35,7 +35,7 @@ jobs: displayName: 'Install docker-cli only' - pwsh: | - C:\Program Files\Docker\Docker\DockerCli.exe -SwitchLinuxEngine + "C:\Program Files\Docker\Docker\DockerCli.exe" -SwitchLinuxEngine displayName: 'Install docker-cli only' - pwsh: | From 4325832f329d776b97e55a74ba7051929cf88d28 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 2 Aug 2021 19:05:43 -0700 Subject: [PATCH 43/90] SwitchDaemon --- eng/pipelines/test-with-proxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index b0be0dd1a950..46a8880a3113 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -35,7 +35,7 @@ jobs: displayName: 'Install docker-cli only' - pwsh: | - "C:\Program Files\Docker\Docker\DockerCli.exe" -SwitchLinuxEngine + "C:\Program Files\Docker\Docker\DockerCli.exe" -SwitchDaemon displayName: 'Install docker-cli only' - pwsh: | From 5ad74f97e0dd94f0cd01aa7fd86e465ae6212534 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 3 Aug 2021 10:53:23 -0700 Subject: [PATCH 44/90] one last attempt with choco --- eng/pipelines/test-with-proxy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 46a8880a3113..c39ca3a02417 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -35,7 +35,8 @@ jobs: displayName: 'Install docker-cli only' - pwsh: | - "C:\Program Files\Docker\Docker\DockerCli.exe" -SwitchDaemon + cd "C:\Program Files\Docker\Docker\" + ./DockerCli.exe -SwitchDaemon displayName: 'Install docker-cli only' - pwsh: | From b267e41c259ef3ab7579bccc3f826d3ddff47da6 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 3 Aug 2021 10:54:05 -0700 Subject: [PATCH 45/90] ok second to last attempt --- eng/pipelines/test-with-proxy.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index c39ca3a02417..b55dbbc9c344 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -38,6 +38,12 @@ jobs: cd "C:\Program Files\Docker\Docker\" ./DockerCli.exe -SwitchDaemon displayName: 'Install docker-cli only' + continueOnError: true + + - pwsh: | + cd "C:\Program Files\Docker\Docker\" + ./DockerCli.exe -SwitchLinuxEngine + displayName: 'Install docker-cli only' - pwsh: | docker run ` From fd84282c6cfc54dc3a9d7bbe283d11966b911030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Mon, 2 Aug 2021 09:47:13 -0700 Subject: [PATCH 46/90] Target AioHttpTransport; async records --- .../devtools_testutils/aio/proxy_testcase_async.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index 22fe4dd117f2..4152dcb58146 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -9,7 +9,7 @@ stop_record_or_playback, ) -from azure.core.pipeline.transport import AsyncioRequestsTransport +from azure.core.pipeline.transport import AioHttpTransport from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function @@ -34,7 +34,7 @@ def transform_args(*args, **kwargs): trimmed_kwargs = {k: v for k, v in kwargs.items()} trim_kwargs_from_test_function(func, trimmed_kwargs) - original_func = AsyncioRequestsTransport.send + original_func = AioHttpTransport.send async def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) @@ -44,13 +44,13 @@ async def combined_call(*args, **kwargs): print("METHOD: ", req.method) return await original_func(*adjusted_args, **adjusted_kwargs) - AsyncioRequestsTransport.send = combined_call + AioHttpTransport.send = combined_call # call the modified function. try: value = await func(*args, **trimmed_kwargs) finally: - AsyncioRequestsTransport.send = original_func + AioHttpTransport.send = original_func stop_record_or_playback(test_id, recording_id) return value From 02731b4c22573727b018f527f63d9e8e0af2cc0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Mon, 9 Aug 2021 16:53:18 -0700 Subject: [PATCH 47/90] Use AZURE_TEST_RUN_LIVE as recording flag --- .../azure_recorded_testcase.py | 13 +++++++++++-- .../devtools_testutils/proxy_testcase.py | 16 ++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py index e15321dadefe..c8e5eb7bc66d 100644 --- a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py @@ -13,11 +13,12 @@ from dotenv import load_dotenv, find_dotenv from azure_devtools.scenario_tests import AzureTestError +from azure_devtools.scenario_tests.config import TestConfig from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function from .config import TEST_SETTING_FILENAME from . import mgmt_settings_fake as fake_settings -from .azure_testcase import is_live, _is_autorest_v3, get_resource_name, get_qualified_method_name +from .azure_testcase import _is_autorest_v3, get_resource_name, get_qualified_method_name try: # Try to import the AsyncFakeCredential, if we cannot assume it is Python 2 @@ -28,6 +29,14 @@ load_dotenv(find_dotenv()) + +def is_live(): + """A module version of is_live, that could be used in pytest marker.""" + if not hasattr(is_live, "_cache"): + is_live._cache = TestConfig().record_mode + return is_live._cache + + class AzureRecordedTestCase(object): @property @@ -60,7 +69,7 @@ def qualified_test_name(self): @property def in_recording(self): - return os.getenv("AZURE_RECORD_MODE") == "record" + return self.is_live # TODO: This needs to be removed, recording processors are handled on the proxy side, but # this is needed for the preparers diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index c01048995bc4..b08fb8bdfce5 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -34,6 +34,10 @@ # this should also fire the admin mapping updates, and start/end the session for commiting recording updates +IS_LIVE = os.getenv("AZURE_TEST_RUN_LIVE") == "true" or os.getenv("AZURE_TEST_RUN_LIVE") == "yes" +IS_PLAYBACK = os.getenv("AZURE_TEST_RUN_LIVE") == "false" or os.getenv("AZURE_TEST_RUN_LIVE") == "no" + + def get_test_id(): # pytest sets the current running test in an environment variable return os.getenv("PYTEST_CURRENT_TEST").split(" ")[0].replace("::", ".") @@ -47,14 +51,14 @@ def get_current_sha(): def start_record_or_playback(test_id): - if os.getenv("AZURE_RECORD_MODE") == "record": + if IS_LIVE: result = requests.post( RECORDING_START_URL, headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, verify=False, ) recording_id = result.headers["x-recording-id"] - elif os.getenv("AZURE_RECORD_MODE") == "playback": + elif IS_PLAYBACK: result = requests.post( PLAYBACK_START_URL, # headers={"x-recording-file": test_id, "x-recording-id": recording_id}, @@ -66,13 +70,13 @@ def start_record_or_playback(test_id): def stop_record_or_playback(test_id, recording_id): - if os.getenv("AZURE_RECORD_MODE") == "record": + if IS_LIVE: result = requests.post( RECORDING_STOP_URL, headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, verify=False, ) - elif os.getenv("AZURE_RECORD_MODE") == "playback": + elif IS_PLAYBACK: result = requests.post( PLAYBACK_STOP_URL, headers={"x-recording-file": test_id, "x-recording-id": recording_id}, @@ -91,13 +95,13 @@ def transform_request(request, recording_id): headers = request.headers # quiet passthrough if neither are set - if os.getenv("AZURE_RECORD_MODE") == "record" or os.getenv("AZURE_RECORD_MODE") == "playback": + if IS_LIVE or IS_PLAYBACK: parsed_result = url_parse.urlparse(request.url) updated_target = parsed_result._replace(**get_proxy_netloc()).geturl() if headers.get("x-recording-upstream-base-uri", None) is None: headers["x-recording-upstream-base-uri"] = "{}://{}".format(parsed_result.scheme, parsed_result.netloc) headers["x-recording-id"] = recording_id - headers["x-recording-mode"] = os.getenv("AZURE_RECORD_MODE") + headers["x-recording-mode"] = "record" if IS_LIVE else "playback" request.url = updated_target From 3f4ef4d64382edd74a830bfb71622c6fd8edb5c1 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:48:55 -0700 Subject: [PATCH 48/90] combine our certificate with the certifi certificate bundle --- .gitignore | 3 ++ scripts/install_certificate.py | 33 +++++++++++++++++++ .../devtools_testutils/proxy_testcase.py | 8 ++--- 3 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 scripts/install_certificate.py diff --git a/.gitignore b/.gitignore index 87b0050a20cc..0e86f299ec3a 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,6 @@ sdk/cosmos/azure-cosmos/test/test_config.py # env vars .env + +# local SSL certificate folder +.certificate \ No newline at end of file diff --git a/scripts/install_certificate.py b/scripts/install_certificate.py new file mode 100644 index 000000000000..d721d607513d --- /dev/null +++ b/scripts/install_certificate.py @@ -0,0 +1,33 @@ +import requests +import os + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..")) +CERT_URL = "https://raw.githubusercontent.com/Azure/azure-sdk-tools/main/tools/test-proxy/docker/dev_certificate/dotnet-devcert.crt" +EXISTING_ROOT_PEM = requests.certs.where() +LOCAL_FILENAME = CERT_URL.split('/')[-1].split(".")[0] + ".pem" +LOCAL_LOCATION = os.path.join(root_dir, '.certificate', LOCAL_FILENAME) + +def download_cert_file(): + content = requests.get(CERT_URL) + + if content.status_code == 200: + with open(LOCAL_LOCATION, 'w') as f: + f.write(content.text) + +def combine_cert_file(): + with open(EXISTING_ROOT_PEM, 'r') as f: + content = f.readlines(); + + with open(LOCAL_LOCATION, 'a') as f: + f.writelines(content) + +if __name__ == "__main__": + download_cert_file() + combine_cert_file() + + print("Set the following certificate paths:") + print("\tSSL_CERT_DIR={}".format(os.path.dirname(LOCAL_LOCATION))) + print("\tREQUESTS_CA_BUNDLE={}".format(LOCAL_LOCATION)) + + print("\nBe aware that REQUESTS_CA_BUNDLE should only be modified ") + diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index b08fb8bdfce5..a9a630ec13d0 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -114,10 +114,10 @@ def transform_args(*args, **kwargs): copied_positional_args = list(args) request = copied_positional_args[1] - # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are - # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get - # a recommendation for how to get a valid SSL Cert for localhost - kwargs["connection_verify"] = False + # # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are + # # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get + # # a recommendation for how to get a valid SSL Cert for localhost + # kwargs["connection_verify"] = False transform_request(request, recording_id) From 8b57e247b408aded6a823cf1bf715713eedbd6e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Mon, 9 Aug 2021 18:38:09 -0700 Subject: [PATCH 49/90] Method for adding sanitizers --- .../tests/test_table_service_stats.py | 5 +- .../devtools_testutils/__init__.py | 2 + .../devtools_testutils/aio/__init__.py | 4 +- .../aio/proxy_testcase_async.py | 16 ++-- .../azure_recorded_testcase.py | 73 +++++++++---------- .../devtools_testutils/config.py | 1 + .../devtools_testutils/enums.py | 11 +++ .../devtools_testutils/helpers.py | 10 +++ .../devtools_testutils/proxy_testcase.py | 39 ++++------ 9 files changed, 87 insertions(+), 74 deletions(-) create mode 100644 tools/azure-sdk-tools/devtools_testutils/enums.py diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py index 8aa0e61075c1..8230ddd06b7f 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from devtools_testutils import AzureRecordedTestCase +from devtools_testutils import AzureRecordedTestCase, ProxyRecordingSanitizer from azure.data.tables import TableServiceClient from _shared.testcase import TableTestCase @@ -12,6 +12,9 @@ # --Test Class ----------------------------------------------------------------- class TestTableServiceStats(AzureRecordedTestCase, TableTestCase): + def setup_method(self): + self.add_sanitizer(ProxyRecordingSanitizer.URI) + # --Test cases per service --------------------------------------- @tables_decorator def test_table_service_stats_f(self, tables_storage_account_name, tables_primary_storage_account_key): diff --git a/tools/azure-sdk-tools/devtools_testutils/__init__.py b/tools/azure-sdk-tools/devtools_testutils/__init__.py index 8a85d5382a55..2c54da0935b5 100644 --- a/tools/azure-sdk-tools/devtools_testutils/__init__.py +++ b/tools/azure-sdk-tools/devtools_testutils/__init__.py @@ -1,6 +1,7 @@ from .mgmt_testcase import AzureMgmtTestCase, AzureMgmtPreparer from .azure_recorded_testcase import AzureRecordedTestCase from .azure_testcase import AzureTestCase, is_live, get_region_override +from .enums import ProxyRecordingSanitizer from .resource_testcase import ( FakeResource, ResourceGroupPreparer, @@ -36,6 +37,7 @@ "RandomNameResourceGroupPreparer", "CachedResourceGroupPreparer", "PowerShellPreparer", + "ProxyRecordingSanitizer", "RecordedByProxy", "ResponseCallback", "RetryCounter", diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py b/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py index 5d6674dc4843..5265d80fe58e 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py @@ -1,5 +1,3 @@ from .proxy_testcase_async import RecordedByProxyAsync -__all__ = [ - "RecordedByProxyAsync" -] +__all__ = ["RecordedByProxyAsync"] diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index 4152dcb58146..1477899c49b6 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -1,7 +1,11 @@ -import asyncio -import functools -from contextlib import contextmanager +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from azure.core.pipeline.transport import AioHttpTransport +from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function from ..proxy_testcase import ( get_test_id, start_record_or_playback, @@ -9,9 +13,6 @@ stop_record_or_playback, ) -from azure.core.pipeline.transport import AioHttpTransport - -from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function def RecordedByProxyAsync(func): async def record_wrap(*args, **kwargs): @@ -39,9 +40,6 @@ def transform_args(*args, **kwargs): async def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) req = adjusted_args[1] - print("HEADERS: ", req.headers) - print("BODY: ", req.body) - print("METHOD: ", req.method) return await original_func(*adjusted_args, **adjusted_kwargs) AioHttpTransport.send = combined_call diff --git a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py index c8e5eb7bc66d..915de1bddfcb 100644 --- a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py @@ -7,8 +7,11 @@ import logging import os import os.path +import requests +import six import sys import time +from typing import TYPE_CHECKING from dotenv import load_dotenv, find_dotenv @@ -16,9 +19,10 @@ from azure_devtools.scenario_tests.config import TestConfig from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function -from .config import TEST_SETTING_FILENAME from . import mgmt_settings_fake as fake_settings from .azure_testcase import _is_autorest_v3, get_resource_name, get_qualified_method_name +from .config import PROXY_URL +from .enums import ProxyRecordingSanitizer try: # Try to import the AsyncFakeCredential, if we cannot assume it is Python 2 @@ -26,6 +30,9 @@ except SyntaxError: pass +if TYPE_CHECKING: + from typing import Optional + load_dotenv(find_dotenv()) @@ -38,16 +45,13 @@ def is_live(): class AzureRecordedTestCase(object): - @property def settings(self): if self.is_live: if self._real_settings: return self._real_settings else: - raise AzureTestError( - "Need a mgmt_settings_real.py file to run tests live." - ) + raise AzureTestError("Need a mgmt_settings_real.py file to run tests live.") else: return self._fake_settings @@ -77,17 +81,26 @@ def in_recording(self): def recording_processors(self): return [] + def add_sanitizer(self, sanitizer, regex=None, value=None): + # type: (ProxyRecordingSanitizer, Optional[str], Optional[str]) -> None + if sanitizer == ProxyRecordingSanitizer.URI: + requests.post( + "{}/Admin/AddSanitizer".format(PROXY_URL), + headers={"x-abstraction-identifier": ProxyRecordingSanitizer.URI.value}, + json={ + "regex": regex or "[a-z]+(?=(?:-secondary)\\.(?:table|blob|queue)\\.core\\.windows\\.net)", + "value": value or "fakevalue" + }, + verify=False + ) + def is_playback(self): return not self.is_live def get_settings_value(self, key): key_value = os.environ.get("AZURE_" + key, None) - if ( - key_value - and self._real_settings - and getattr(self._real_settings, key) != key_value - ): + if key_value and self._real_settings and getattr(self._real_settings, key) != key_value: raise ValueError( "You have both AZURE_{key} env variable and mgmt_settings_real.py for {key} to different values".format( key=key @@ -97,22 +110,14 @@ def get_settings_value(self, key): if not key_value: try: key_value = getattr(self.settings, key) - except Exception: - print("Could not get {}".format(key)) - raise + except Exception as ex: + six.raise_from(ValueError("Could not get {}".format(key)), ex) return key_value def get_credential(self, client_class, **kwargs): - - tenant_id = os.environ.get( - "AZURE_TENANT_ID", getattr(self._real_settings, "TENANT_ID", None) - ) - client_id = os.environ.get( - "AZURE_CLIENT_ID", getattr(self._real_settings, "CLIENT_ID", None) - ) - secret = os.environ.get( - "AZURE_CLIENT_SECRET", getattr(self._real_settings, "CLIENT_SECRET", None) - ) + tenant_id = os.environ.get("AZURE_TENANT_ID", getattr(self._real_settings, "TENANT_ID", None)) + client_id = os.environ.get("AZURE_CLIENT_ID", getattr(self._real_settings, "CLIENT_ID", None)) + secret = os.environ.get("AZURE_CLIENT_SECRET", getattr(self._real_settings, "CLIENT_SECRET", None)) is_async = kwargs.pop("is_async", False) if tenant_id and client_id and secret and self.is_live: @@ -122,24 +127,21 @@ def get_credential(self, client_class, **kwargs): if is_async: from azure.identity.aio import ClientSecretCredential - return ClientSecretCredential( - tenant_id=tenant_id, client_id=client_id, client_secret=secret - ) + return ClientSecretCredential(tenant_id=tenant_id, client_id=client_id, client_secret=secret) else: # Create msrestazure class from msrestazure.azure_active_directory import ( ServicePrincipalCredentials, ) - return ServicePrincipalCredentials( - tenant=tenant_id, client_id=client_id, secret=secret - ) + return ServicePrincipalCredentials(tenant=tenant_id, client_id=client_id, secret=secret) else: if _is_autorest_v3(client_class): if is_async: if self.is_live: raise ValueError( - "Async live doesn't support mgmt_setting_real, please set AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET" + "Async live doesn't support mgmt_setting_real, please set AZURE_TENANT_ID," + "AZURE_CLIENT_ID, AZURE_CLIENT_SECRET" ) return AsyncFakeCredential() else: @@ -161,9 +163,7 @@ def create_client_from_credential(self, client_class, credential, **kwargs): if self.is_playback(): try: - client._config.polling_interval = ( - 0 # FIXME in azure-mgmt-core, make this a kwargs - ) + client._config.polling_interval = 0 # FIXME in azure-mgmt-core, make this a kwargs except AttributeError: pass @@ -186,7 +186,6 @@ def create_basic_client(self, client_class, **kwargs): def create_random_name(self, name): unique_test_name = os.getenv("PYTEST_CURRENT_TEST").encode("utf-8") - print(unique_test_name) return get_resource_name(name, unique_test_name) def get_resource_name(self, name): @@ -194,7 +193,7 @@ def get_resource_name(self, name): return self.create_random_name(name) def get_replayable_random_resource_name(self, name): - """In a replay scenario, (is not live) gives the static moniker. In the random scenario, gives generated name.""" + """In a replay scenario (not live), gives the static moniker. In the random scenario, gives generated name.""" if self.is_live: created_name = self.create_random_name(name) self.scrubber.register_name_pair(created_name, name) @@ -205,9 +204,7 @@ def get_preparer_resource_name(self, prefix): If prefix is a blank string, use the fully qualified test name instead. This is what legacy tests do for resource groups.""" - return self.get_resource_name( - prefix #or self.qualified_test_name.replace(".", "_") - ) + return self.get_resource_name(prefix) # or self.qualified_test_name.replace(".", "_") @staticmethod def await_prepared_test(test_fn): diff --git a/tools/azure-sdk-tools/devtools_testutils/config.py b/tools/azure-sdk-tools/devtools_testutils/config.py index 117c3b38ca08..27b4efd90d6a 100644 --- a/tools/azure-sdk-tools/devtools_testutils/config.py +++ b/tools/azure-sdk-tools/devtools_testutils/config.py @@ -1 +1,2 @@ +PROXY_URL = "http://localhost:5000" TEST_SETTING_FILENAME = "testsettings_local.cfg" diff --git a/tools/azure-sdk-tools/devtools_testutils/enums.py b/tools/azure-sdk-tools/devtools_testutils/enums.py new file mode 100644 index 000000000000..f1456dbc8a1f --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/enums.py @@ -0,0 +1,11 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from enum import Enum + +class ProxyRecordingSanitizer(str, Enum): + """General-purpose sanitizers for sanitizing test proxy recordings""" + + URI = "UriRegexSanitizer" diff --git a/tools/azure-sdk-tools/devtools_testutils/helpers.py b/tools/azure-sdk-tools/devtools_testutils/helpers.py index e751e09cbd55..00df86a6332d 100644 --- a/tools/azure-sdk-tools/devtools_testutils/helpers.py +++ b/tools/azure-sdk-tools/devtools_testutils/helpers.py @@ -3,6 +3,16 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +from azure_devtools.scenario_tests.config import TestConfig + + +def is_live(): + """A module version of is_live, that could be used in pytest marker.""" + if not hasattr(is_live, "_cache"): + is_live._cache = TestConfig().record_mode + return is_live._cache + + class RetryCounter(object): def __init__(self): self.count = 0 diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index a9a630ec13d0..40bf261b8541 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- - import os import requests @@ -21,9 +20,11 @@ # the trimming function to clean up incoming arguments to the test function we are wrapping from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function +from devtools_testutils.azure_recorded_testcase import is_live +from .config import PROXY_URL + # defaults -PROXY_URL = "http://localhost:5000" RECORDING_START_URL = "{}/record/start".format(PROXY_URL) RECORDING_STOP_URL = "{}/record/stop".format(PROXY_URL) PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL) @@ -34,10 +35,6 @@ # this should also fire the admin mapping updates, and start/end the session for commiting recording updates -IS_LIVE = os.getenv("AZURE_TEST_RUN_LIVE") == "true" or os.getenv("AZURE_TEST_RUN_LIVE") == "yes" -IS_PLAYBACK = os.getenv("AZURE_TEST_RUN_LIVE") == "false" or os.getenv("AZURE_TEST_RUN_LIVE") == "no" - - def get_test_id(): # pytest sets the current running test in an environment variable return os.getenv("PYTEST_CURRENT_TEST").split(" ")[0].replace("::", ".") @@ -51,14 +48,14 @@ def get_current_sha(): def start_record_or_playback(test_id): - if IS_LIVE: + if is_live: result = requests.post( RECORDING_START_URL, headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, verify=False, ) recording_id = result.headers["x-recording-id"] - elif IS_PLAYBACK: + else: result = requests.post( PLAYBACK_START_URL, # headers={"x-recording-file": test_id, "x-recording-id": recording_id}, @@ -70,14 +67,14 @@ def start_record_or_playback(test_id): def stop_record_or_playback(test_id, recording_id): - if IS_LIVE: - result = requests.post( + if is_live: + requests.post( RECORDING_STOP_URL, headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, verify=False, ) - elif IS_PLAYBACK: - result = requests.post( + else: + requests.post( PLAYBACK_STOP_URL, headers={"x-recording-file": test_id, "x-recording-id": recording_id}, verify=False, @@ -91,18 +88,15 @@ def get_proxy_netloc(): def transform_request(request, recording_id): """Redirect the request to the test proxy, and store the original request URI in a header""" - upstream_url = request.url.replace("//text", "/text") headers = request.headers - # quiet passthrough if neither are set - if IS_LIVE or IS_PLAYBACK: - parsed_result = url_parse.urlparse(request.url) - updated_target = parsed_result._replace(**get_proxy_netloc()).geturl() - if headers.get("x-recording-upstream-base-uri", None) is None: - headers["x-recording-upstream-base-uri"] = "{}://{}".format(parsed_result.scheme, parsed_result.netloc) - headers["x-recording-id"] = recording_id - headers["x-recording-mode"] = "record" if IS_LIVE else "playback" - request.url = updated_target + parsed_result = url_parse.urlparse(request.url) + updated_target = parsed_result._replace(**get_proxy_netloc()).geturl() + if headers.get("x-recording-upstream-base-uri", None) is None: + headers["x-recording-upstream-base-uri"] = "{}://{}".format(parsed_result.scheme, parsed_result.netloc) + headers["x-recording-id"] = recording_id + headers["x-recording-mode"] = "record" if is_live else "playback" + request.url = updated_target def RecordedByProxy(func): @@ -140,7 +134,6 @@ def combined_call(*args, **kwargs): value = func(*args, **trimmed_kwargs) finally: RequestsTransport.send = original_transport_func - # print("Exiting patch context. RequestsTransport.send is at {}".format(id(RequestsTransport.send))) stop_record_or_playback(test_id, recording_id) return value From a4971c597cde402a50226a13ca29cc7823932a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Wed, 11 Aug 2021 17:48:27 -0700 Subject: [PATCH 50/90] Default to HTTPS --- .../tests/test_table_service_stats.py | 2 +- .../devtools_testutils/aio/proxy_testcase_async.py | 7 +------ .../devtools_testutils/azure_recorded_testcase.py | 3 +-- tools/azure-sdk-tools/devtools_testutils/config.py | 9 ++++++++- tools/azure-sdk-tools/devtools_testutils/helpers.py | 8 -------- .../devtools_testutils/proxy_testcase.py | 13 ++----------- 6 files changed, 13 insertions(+), 29 deletions(-) diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py index 8230ddd06b7f..f76c5144c2aa 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py @@ -13,7 +13,7 @@ class TestTableServiceStats(AzureRecordedTestCase, TableTestCase): def setup_method(self): - self.add_sanitizer(ProxyRecordingSanitizer.URI) + self.add_sanitizer(ProxyRecordingSanitizer.URI, value="faketableaccount") # --Test cases per service --------------------------------------- @tables_decorator diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index 1477899c49b6..71d468360e01 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -23,11 +23,6 @@ def transform_args(*args, **kwargs): copied_positional_args = list(args) request = copied_positional_args[1] - # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are - # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get - # a recommendation for how to get a valid SSL Cert for localhost - kwargs["connection_verify"] = False - transform_request(request, recording_id) return tuple(copied_positional_args), kwargs @@ -39,7 +34,7 @@ def transform_args(*args, **kwargs): async def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) - req = adjusted_args[1] + adjusted_args[1] return await original_func(*adjusted_args, **adjusted_kwargs) AioHttpTransport.send = combined_call diff --git a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py index 915de1bddfcb..e045da62966a 100644 --- a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py @@ -91,7 +91,6 @@ def add_sanitizer(self, sanitizer, regex=None, value=None): "regex": regex or "[a-z]+(?=(?:-secondary)\\.(?:table|blob|queue)\\.core\\.windows\\.net)", "value": value or "fakevalue" }, - verify=False ) def is_playback(self): @@ -204,7 +203,7 @@ def get_preparer_resource_name(self, prefix): If prefix is a blank string, use the fully qualified test name instead. This is what legacy tests do for resource groups.""" - return self.get_resource_name(prefix) # or self.qualified_test_name.replace(".", "_") + return self.get_resource_name(prefix) @staticmethod def await_prepared_test(test_fn): diff --git a/tools/azure-sdk-tools/devtools_testutils/config.py b/tools/azure-sdk-tools/devtools_testutils/config.py index 27b4efd90d6a..7b9bcfbb06aa 100644 --- a/tools/azure-sdk-tools/devtools_testutils/config.py +++ b/tools/azure-sdk-tools/devtools_testutils/config.py @@ -1,2 +1,9 @@ -PROXY_URL = "http://localhost:5000" +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + + +PROXY_URL = "https://localhost:5001" TEST_SETTING_FILENAME = "testsettings_local.cfg" diff --git a/tools/azure-sdk-tools/devtools_testutils/helpers.py b/tools/azure-sdk-tools/devtools_testutils/helpers.py index 00df86a6332d..4d0867aea542 100644 --- a/tools/azure-sdk-tools/devtools_testutils/helpers.py +++ b/tools/azure-sdk-tools/devtools_testutils/helpers.py @@ -3,14 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from azure_devtools.scenario_tests.config import TestConfig - - -def is_live(): - """A module version of is_live, that could be used in pytest marker.""" - if not hasattr(is_live, "_cache"): - is_live._cache = TestConfig().record_mode - return is_live._cache class RetryCounter(object): diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 40bf261b8541..7f8eb78ad4dd 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -20,8 +20,8 @@ # the trimming function to clean up incoming arguments to the test function we are wrapping from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function -from devtools_testutils.azure_recorded_testcase import is_live from .config import PROXY_URL +from .helpers import is_live # defaults @@ -52,7 +52,6 @@ def start_record_or_playback(test_id): result = requests.post( RECORDING_START_URL, headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, - verify=False, ) recording_id = result.headers["x-recording-id"] else: @@ -60,7 +59,6 @@ def start_record_or_playback(test_id): PLAYBACK_START_URL, # headers={"x-recording-file": test_id, "x-recording-id": recording_id}, headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, - verify=False, ) recording_id = result.headers["x-recording-id"] return recording_id @@ -71,13 +69,11 @@ def stop_record_or_playback(test_id, recording_id): requests.post( RECORDING_STOP_URL, headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, - verify=False, ) else: requests.post( PLAYBACK_STOP_URL, headers={"x-recording-file": test_id, "x-recording-id": recording_id}, - verify=False, ) @@ -108,11 +104,6 @@ def transform_args(*args, **kwargs): copied_positional_args = list(args) request = copied_positional_args[1] - # # TODO, get the test-proxy server a real SSL certificate. The issue here is that SSL Certificates are - # # normally associated with a domain name. Need to talk to the //SSLAdmin folks (or someone else) and get - # # a recommendation for how to get a valid SSL Cert for localhost - # kwargs["connection_verify"] = False - transform_request(request, recording_id) return tuple(copied_positional_args), kwargs @@ -124,7 +115,7 @@ def transform_args(*args, **kwargs): def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) - req = adjusted_args[1] + adjusted_args[1] return original_transport_func(*adjusted_args, **adjusted_kwargs) RequestsTransport.send = combined_call From 53b4d14e56fe7cc6ec56bcaf518da9ec41eb4f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Wed, 11 Aug 2021 19:26:08 -0700 Subject: [PATCH 51/90] Detect is_live correctly --- .../devtools_testutils/aio/proxy_testcase_async.py | 1 - .../azure-sdk-tools/devtools_testutils/proxy_testcase.py | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index 71d468360e01..11c8f9e44395 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -34,7 +34,6 @@ def transform_args(*args, **kwargs): async def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) - adjusted_args[1] return await original_func(*adjusted_args, **adjusted_kwargs) AioHttpTransport.send = combined_call diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 7f8eb78ad4dd..be28b35b41ca 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -20,8 +20,8 @@ # the trimming function to clean up incoming arguments to the test function we are wrapping from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function +from .azure_recorded_testcase import is_live from .config import PROXY_URL -from .helpers import is_live # defaults @@ -48,7 +48,7 @@ def get_current_sha(): def start_record_or_playback(test_id): - if is_live: + if is_live(): result = requests.post( RECORDING_START_URL, headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, @@ -65,7 +65,7 @@ def start_record_or_playback(test_id): def stop_record_or_playback(test_id, recording_id): - if is_live: + if is_live(): requests.post( RECORDING_STOP_URL, headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, @@ -91,7 +91,7 @@ def transform_request(request, recording_id): if headers.get("x-recording-upstream-base-uri", None) is None: headers["x-recording-upstream-base-uri"] = "{}://{}".format(parsed_result.scheme, parsed_result.netloc) headers["x-recording-id"] = recording_id - headers["x-recording-mode"] = "record" if is_live else "playback" + headers["x-recording-mode"] = "record" if is_live() else "playback" request.url = updated_target @@ -115,7 +115,6 @@ def transform_args(*args, **kwargs): def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) - adjusted_args[1] return original_transport_func(*adjusted_args, **adjusted_kwargs) RequestsTransport.send = combined_call From 27e77dafccdcee7ae72035d485a74f08614b52d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Wed, 11 Aug 2021 21:16:49 -0700 Subject: [PATCH 52/90] Cleaned proxy_testcase.py --- tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index be28b35b41ca..6ab7d75e64d4 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -57,7 +57,6 @@ def start_record_or_playback(test_id): else: result = requests.post( PLAYBACK_START_URL, - # headers={"x-recording-file": test_id, "x-recording-id": recording_id}, headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, ) recording_id = result.headers["x-recording-id"] From 3ef32536b441dedbe571b3ba533e49fbee9abc6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Thu, 12 Aug 2021 10:50:26 -0700 Subject: [PATCH 53/90] Hacky way to store recordings correctly --- ...estTableServiceStats.test_table_service_stats_f | 14 +++++++------- .../devtools_testutils/proxy_testcase.py | 6 +++++- 2 files changed, 12 insertions(+), 8 deletions(-) rename sdk/tables/azure-data-tables/tests/recordings/{sdk/tables/azure-data-tables/tests => }/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f (73%) diff --git a/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f b/sdk/tables/azure-data-tables/tests/recordings/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f similarity index 73% rename from sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f rename to sdk/tables/azure-data-tables/tests/recordings/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f index 1a1a0b641a94..b42c6f18aa45 100644 --- a/sdk/tables/azure-data-tables/tests/recordings/sdk/tables/azure-data-tables/tests/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f +++ b/sdk/tables/azure-data-tables/tests/recordings/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f @@ -8,27 +8,27 @@ "Accept-Encoding": "gzip, deflate", "Authorization": "Sanitized", "Connection": "keep-alive", - "Date": "Wed, 21 Jul 2021 22:10:01 GMT", + "Date": "Thu, 12 Aug 2021 17:45:09 GMT", "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.0 (Windows-10-10.0.19041-SP0)", - "x-ms-client-request-id": "652f28dd-ea70-11eb-8b90-c8348e51ab51", - "x-ms-date": "Wed, 21 Jul 2021 22:10:01 GMT", + "x-ms-client-request-id": "09dbc5e2-fb95-11eb-a228-c8348e51ab51", + "x-ms-date": "Thu, 12 Aug 2021 17:45:09 GMT", "x-ms-version": "2019-02-02" }, "RequestBody": null, "StatusCode": 200, "ResponseHeaders": { "Content-Type": "application/xml", - "Date": "Wed, 21 Jul 2021 22:10:01 GMT", + "Date": "Thu, 12 Aug 2021 17:45:08 GMT", "Server": [ "Windows-Azure-Table/1.0", "Microsoft-HTTPAPI/2.0" ], "Transfer-Encoding": "chunked", - "x-ms-client-request-id": "652f28dd-ea70-11eb-8b90-c8348e51ab51", - "x-ms-request-id": "453e627a-3002-0023-147d-7e288a000000", + "x-ms-client-request-id": "09dbc5e2-fb95-11eb-a228-c8348e51ab51", + "x-ms-request-id": "a7683a07-2002-002f-0ba1-8fbf82000000", "x-ms-version": "2019-02-02" }, - "ResponseBody": "\uFEFF\u003C?xml version=\u00221.0\u0022 encoding=\u0022utf-8\u0022?\u003E\u003CStorageServiceStats\u003E\u003CGeoReplication\u003E\u003CStatus\u003Elive\u003C/Status\u003E\u003CLastSyncTime\u003EWed, 21 Jul 2021 22:06:46 GMT\u003C/LastSyncTime\u003E\u003C/GeoReplication\u003E\u003C/StorageServiceStats\u003E" + "ResponseBody": "\uFEFF\u003C?xml version=\u00221.0\u0022 encoding=\u0022utf-8\u0022?\u003E\u003CStorageServiceStats\u003E\u003CGeoReplication\u003E\u003CStatus\u003Elive\u003C/Status\u003E\u003CLastSyncTime\u003EThu, 12 Aug 2021 17:42:04 GMT\u003C/LastSyncTime\u003E\u003C/GeoReplication\u003E\u003C/StorageServiceStats\u003E" } ], "Variables": {} diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 6ab7d75e64d4..44667c1bf165 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -37,7 +37,11 @@ def get_test_id(): # pytest sets the current running test in an environment variable - return os.getenv("PYTEST_CURRENT_TEST").split(" ")[0].replace("::", ".") + path_to_test = os.getenv("PYTEST_CURRENT_TEST").split(" ")[0].replace("::", ".") + beginning, end = path_to_test.split("/tests/") + full = beginning + "/tests/recordings/" + end + print(full) + return full def get_current_sha(): From 86b35fe6dd84c698f9c03e27751a78eab9258591 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 12 Aug 2021 17:27:59 -0700 Subject: [PATCH 54/90] preparing for tests --- eng/common/scripts/common.ps1 | 1 + eng/common/testproxy/docker-start-proxy.ps1 | 83 ++++++++++++++ eng/common/testproxy/dotnet-devcert.crt | 20 ++++ eng/common/testproxy/dotnet-devcert.pfx | Bin 0 -> 2445 bytes eng/common/testproxy/test-proxy-docker.yml | 16 +++ eng/common/testproxy/test-proxy-tool.yml | 36 +++++++ .../templates/steps/check-availability.yml | 8 ++ eng/pipelines/test-with-proxy.yml | 101 +++++++----------- eng/scripts/Language-Settings.ps1 | 5 + 9 files changed, 206 insertions(+), 64 deletions(-) create mode 100644 eng/common/testproxy/docker-start-proxy.ps1 create mode 100644 eng/common/testproxy/dotnet-devcert.crt create mode 100644 eng/common/testproxy/dotnet-devcert.pfx create mode 100644 eng/common/testproxy/test-proxy-docker.yml create mode 100644 eng/common/testproxy/test-proxy-tool.yml create mode 100644 eng/pipelines/templates/steps/check-availability.yml diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 4e0b0847cdbf..5703b16d04d3 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -44,3 +44,4 @@ $GetDocsMsMetadataForPackageFn = "Get-${Language}-DocsMsMetadataForPackage" $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageSpecificPackageInfo" $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview" +$TestProxyTrustCertFn = "Import-Dev-Cert-${Language}" \ No newline at end of file diff --git a/eng/common/testproxy/docker-start-proxy.ps1 b/eng/common/testproxy/docker-start-proxy.ps1 new file mode 100644 index 000000000000..4e5da37278a2 --- /dev/null +++ b/eng/common/testproxy/docker-start-proxy.ps1 @@ -0,0 +1,83 @@ + #!/usr/bin/env pwsh -c + +<# +.DESCRIPTION +Start the docker proxy container. If it is already running, quietly exit. Any other error should fail. +.PARAMETER Mode +"start" or "stop" to start up or stop the test-proxy instance. +.PARAMETER TargetFolder +The folder in which context the test proxy will be started. Defaults to current working directory. +#> +[CmdletBinding(SupportsShouldProcess = $true)] +param( + [ValidateSet("start", "stop")] + [String] + $Mode, + [String] + $TargetFolder = "." +) + +try { + docker --version | Out-Null +} +catch { + Write-Error "A invocation of docker --version failed. This indicates that docker is not properly installed or running." + Write-Error "Please check your docker invocation and try running the script again." +} + +$SELECTED_IMAGE_TAG = "1037115" +$CONTAINER_NAME = "ambitious_azsdk_test_proxy" +$LINUX_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-lin:${SELECTED_IMAGE_TAG}" +$WINDOWS_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-win:${SELECTED_IMAGE_TAG}" +$root = (Resolve-Path $TargetFolder).Path.Replace("`\", "/") + +function Get-Proxy-Container(){ + return (docker container ls -a --format "{{ json . }}" --filter "name=$CONTAINER_NAME" ` + | ConvertFrom-Json ` + | Select-Object -First 1) +} + + +$SelectedImage = $LINUX_IMAGE_SOURCE +$Initial = "" + +# most of the time, running this script on a windows machine will work just fine, as docker defaults to linux containers +# however, in CI, windows images default to _windows_ containers. We cannot swap them. We can tell if we're in a CI build by +# checking for the environment variable TF_BUILD. +if ($IsWindows -and $env:TF_BUILD){ + $SelectedImage = $WINDOWS_IMAGE_SOURCE + $Initial = "C:" +} + +if ($Mode -eq "start"){ + $proxyContainer = Get-Proxy-Container + + # if we already have one, we just need to check the state + if($proxyContainer){ + if ($proxyContainer.State -eq "running") + { + Write-Host "Discovered an already running instance of the test-proxy!. Exiting" + exit(0) + } + } + # else we need to create it + else { + Write-Host "Attempting creation of Docker host $CONTAINER_NAME" + Write-Host "docker container create -v `"${root}:${Initial}/etc/testproxy`" -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage" + docker container create -v "${root}:${Initial}/etc/testproxy" -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage + } + + Write-Host "Attempting start of Docker host $CONTAINER_NAME" + docker container start $CONTAINER_NAME +} + +if ($Mode -eq "stop"){ + $proxyContainer = Get-Proxy-Container + + if($proxyContainer){ + if($proxyContainer.State -eq "running"){ + Write-Host "Found a running instance of $CONTAINER_NAME, shutting it down." + docker container stop $CONTAINER_NAME + } + } +} \ No newline at end of file diff --git a/eng/common/testproxy/dotnet-devcert.crt b/eng/common/testproxy/dotnet-devcert.crt new file mode 100644 index 000000000000..e8575ea44564 --- /dev/null +++ b/eng/common/testproxy/dotnet-devcert.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSDCCAjCgAwIBAgIUPMKpJ/j10eQrcQBNnkImIaOYHakwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIxMDgwNTAwMzU1NloXDTIyMDgw +NTAwMzU1NlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAxe/ZseXgOTVoF7uTjX5Leknk95jIoyGc+VlxA8BhzGOr +r4u6VNQZRCMq+svHY36tW4+u/xHNe2kvbwy2mnS8cFFLfst+94qBZVJDBxSGZ9I/ +wekErNsjFsik4UrMvcC+ZlGPh7hb3f7tSx29tn1DIkAUXVnbZ6TT5s+mYRQpZ6fW +6kR3RNfc0A1IUM7Zs9yfNEr0O2H41P2HcLKoOPtvd7GvTQm9Ofh3srKvII+sZn/J +WH7r76oRQMX904mOMdryQwZLObsqX4dXIEbafKVSecB3PBVIhv8gVtJhcZbQP1pI +mMiWd6PHv46ZhGf7+cKnYUSa8Ia2t/wetK1wd00dFwIDAQABo4GRMIGOMA8GA1Ud +EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGmMBYGA1UdJQEB/wQMMAoGCCsGAQUF +BwMBMBcGA1UdEQEB/wQNMAuCCWxvY2FsaG9zdDA6BgorBgEEAYI3VAEBBCwMKkFT +UC5ORVQgQ29yZSBIVFRQUyBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTANBgkqhkiG +9w0BAQsFAAOCAQEAIj2VlBVcXGSly6KCBg6lgwFi+henWfSox77iuGAaAxDjN3jd +9lZahW4MPNLHKSrPRb4YNSLZ2jh7zdcttQrqd4qH65o1q56q5JrCmli99iIzY9Y8 +RdYyxK4Zzr31wjpsyFiWQfqJTuSFUUg9uDDj0negwEZLIGlt7nr12wflt2+QOJtD +byMeSZLbB5dPzn341DK0qfJEJMMgL0XsPEVZ3TQ6Alc9zq5wI608C/mXnz3xJE05 +UTYD8pRJJ/DyG0empvOVE8Sg93msHPquAbgqO9aqCpykgg/a8CFvI4wRdfvGEFlv +8XJKL8Y/PFsmFeO3axq3zUYKFVdc9Un4dFIaag== +-----END CERTIFICATE----- diff --git a/eng/common/testproxy/dotnet-devcert.pfx b/eng/common/testproxy/dotnet-devcert.pfx new file mode 100644 index 0000000000000000000000000000000000000000..28058ae4ce30e7a413e8f872d930fa2cf5c2107c GIT binary patch literal 2445 zcmY+Ec{me}AIEKGOw3W1T%&R;L(Z@fa?O=93FS=q=BQl7a-{|M|6Xqd27(0wI1w0#Ef}JdY@Ym$AHWSL zz(7zS3`GAI)K?F!Tb zfjk`X7|v1{OYWP-ZccVCBTB-xZYA4g%!tC;_FsW8vbMzZGh3`8ogVrPMR>ljPU4#- z`G@P4g@lf03(hTENs)vkx^jBoGHwj2avv-Dpe_DO>@3jk<%U$iX9rZd`=jP! zsg>})<%?TyfZg|Y*>jhF5Sk^W#Xi;Zf992<-1wem2j0S$iBLn3j~5NK^GDtfIXo`5 zEs4Sf`)CEeP0j%@`i~ke|2fh@=L=@SIJ}`Y+kWWchubBzbXCRXNLitiiMY3dmu!a( zFZPEvT^=8TlPT8*{GH8Y<-2>g>eDZprB2Z+>uP-Z?YAXY9h_{s@4_XehsU4Id_iyC zJ+N--k7cI{U3Bi+p2UKd+buQ)=Z*I*(W5Hw)&cuE;drYzBT8WE$tY9^^4o53%-qi5 z=n{`d!C!c+~BNXhVHGYMn+(JJ0#LCDfvOFPO#fMYz)uYiLDW|Co z(JGTK!;W~b1WLVc_OQ;;y-e`1>Itds$%e}1YQK|Ze#ae*l3sb6q>JipNOI#XV=WLu zsiaZOG)Y@E9Hc?8IGLp#_fdMEJxgwAltLxG(ZXfxQMhi>=<_k}P;!U4_5&K0su_n9 zQ&wnVaA_R&m2zs__fZs>G)%_NW)SfLFgowbEDRqBT^z zJMo)_(u@dc8+TU{Xdrq5=H(TjX}T`cCNE>>pH3;Ad}l6&(NCGu#)3`Gm)tWnFX*fWw>iusH-DeXc-0_K%B?cPcYb`dW?L5*s2f4hp zKeP|&@yDS&;8*PpQw~cQEhAf!7vf#A2DL4a-dY3E@O)XZKK~Y*%(FLu`*?(Zen_F2 zPxvUY0RHLI{s$pqq*$-r#fUBwfZo(KCs%hX?u@6-^jp@|GJ(4$>d|yLeaX7K7qf1N25=t-MWP)nc}xHt_coMNUZF&!y2dObegcrn^uF-D>4qs;DA=zqU?bmyx|I zx{YWUS(>xDdPsmyh{Z*-lg^&5RYzXkyVj#%Jjao_N!q@F2y*i=D;iN=%^8j1li)|W zO$Tp!3PT4fpfxfDq zUhF8y%X|XI1;l*cygss_+G?*f_ik^z=Ke5}=A~KOvHxYg)!i}dxwR25Xf+f!SPifN zK_Y9AKWpP+oI@T^$XT5`{_B4e3fD|c@Q}89T*QQwyld*zXZCAD4>GC}9;NG;RG0(5 znCLlG-=@yqe0I($$p5%?wp-oxY5FwGxwa%0!TBYiZX~;QPV~q@A%SXMowq=`bW3rT zPIq&knIrEPBS8(i=47qf){UF!GGy&r0Z;BbJc~h^^*-$0zZUMj%x?59QtrM0%^JKZ$4FPS{j?Ut<*pX!#dB%!EbOGU5*$5VFUa)Y@$M;Z z4sl8Ns(swP-zX*T#^Q9`@)Sed>bf4v-3@?4?C!~cisATi`B3dvT1mth0W%(+%O^(X)3 zB7ad%T0XtsUdE>o28`a;_iQtx4RfXR|6{`h;drtDABlWO*H zgT|>(isc2UO4D+-y-(X5*;PMlIb6=BB$EqoGaR;5T|(3cEW!exbkWR0 zKSxCTZ>!9^nIMV@rXZp3&~tT3_6(~U@dzr@{&LObWzJXp+%8Ugw8q*?g_nMorpp7&E#Ztx{^h3z}&B?D+fl@MlU+)b>qDi^1L1WD%$SyN(bMH5llaAfgwMN+(1yb)GL# zB644QI=;pr)DSWV2$)M5$_W(X;s71nM&EriZMXu9q4}jIu`Zb2ZMe}2;=yh6dp0e8 JEc(5Y{{n}~oSOgu literal 0 HcmV?d00001 diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml new file mode 100644 index 000000000000..a86944ac83e8 --- /dev/null +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -0,0 +1,16 @@ +parameters: + rootFolder: '$(Build.SourcesDirectory)' + +steps: + - pwsh: | + . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 + + if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) + { + &$FindArtifactForApiReviewFn + } + displayName: 'Language Specific Certificate Trust' + + - pwsh: | + $(Build.SourcesDirectory)/eng/common/scripts/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" + displayName: 'Run the docker container' diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml new file mode 100644 index 000000000000..78a6e2b8d7c7 --- /dev/null +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -0,0 +1,36 @@ +parameters: + rootFolder: '$(Build.SourcesDirectory)' + +steps: + - pwsh: | + . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 + + if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) + { + &$FindArtifactForApiReviewFn + } + displayName: 'Language Specific Certificate Trust' + + - pwsh: | + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]$(Build.SourcesDirectory)/eng/common/testproxy/dotnet-devcert.pfx" + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password" + displayName: 'Configure Kestrel Environment Variables' + + - pwsh: | + dotnet tool install azure.sdk.tools.testproxy ` + --tool-path $(Build.BinariesDirectory)/test-proxy ` + --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json ` + --version 1.0.0-dev.20210811.2 + displayName: "Install test-proxy" + + - pwsh: | + if ($IsWindows) { + Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` + -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` + -NoNewWindow -PassThru + } + else { + nohup $(Build.BinariesDirectory)/test-proxy/test-proxy ` + --storage-location "${{ parameters.rootFolder }}" & + } + displayName: 'Run the testproxy via dotnet tool' \ No newline at end of file diff --git a/eng/pipelines/templates/steps/check-availability.yml b/eng/pipelines/templates/steps/check-availability.yml new file mode 100644 index 000000000000..01d24a283739 --- /dev/null +++ b/eng/pipelines/templates/steps/check-availability.yml @@ -0,0 +1,8 @@ +steps: + - pwsh: | + Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" + displayName: 'http://localhost:5000/Info/Available' + + - pwsh: | + Invoke-WebRequest -Uri "https://localhost:5001/Info/Available" -SkipCertificateCheck + displayName: 'https://localhost:5001/Info/Available' \ No newline at end of file diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index b55dbbc9c344..dba95f0f67b3 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -4,77 +4,50 @@ variables: PythonVersion: '3.9' jobs: - - job: 'rpat' - displayName: 'Run Proxy and Test' - + - job: 'win2019docker' + displayName: 'Windows Docker Run' pool: vmImage: 'windows-2019' - steps: - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) + - template: /eng/common/testproxy-test-proxy-docker.yml + - template : /eng/pipelines/templates/steps/check-availability.yml - - pwsh: | - choco --version - displayName: "Is Choco available?" - continueOnError: true - - - pwsh: | - dir env: - displayName: "Dump Env" - - - pwsh: | - Get-ChildItem -Recurse "C:/Program Files/Docker" - displayName: 'Dump Details of Docker Install' - continueOnError: true - - - pwsh: | - choco install docker-desktop - displayName: 'Install docker-cli only' - - - pwsh: | - cd "C:\Program Files\Docker\Docker\" - ./DockerCli.exe -SwitchDaemon - displayName: 'Install docker-cli only' - continueOnError: true - - - pwsh: | - cd "C:\Program Files\Docker\Docker\" - ./DockerCli.exe -SwitchLinuxEngine - displayName: 'Install docker-cli only' - - - pwsh: | - docker run ` - --detach ` - -v $(Build.SourcesDirectory):/etc/testproxy ` - -p 5001:5001 ` - -p 5000:5000 ` - azsdkengsys.azurecr.io/engsys/ubuntu_testproxy_server:latest - displayName: 'Run the docker container' - - - pwsh: | - docker container ls -a - displayName: "View running docker containers" - continueOnError: true - - - pwsh: | - Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" - displayName: 'http://localhost:5000/Info/Available' - continueOnError: true - - - pwsh: | - Invoke-WebRequest -Uri "https://localhost:5000/Info/Available" - displayName: 'https://localhost:5000/Info/Available' - continueOnError: true + - job: 'win2019tool' + displayName: 'Windows Tool Run' + pool: + vmImage: 'windows-2019' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy-test-proxy-docker.yml + - template : /eng/pipelines/templates/steps/check-availability.yml - - pwsh: | - Invoke-WebRequest -Uri "http://localhost:5001/Info/Available" - displayName: 'http://localhost:5001/Info/Available' - continueOnError: true + - job: 'linux2020docker' + displayName: 'Linux Docker Run' + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy-test-proxy-docker.yml + - template : /eng/pipelines/templates/steps/check-availability.yml - - pwsh: | - Invoke-WebRequest -Uri "https://localhost:5001/Info/Available" - displayName: 'https://localhost:5001/Info/Available' - continueOnError: true \ No newline at end of file + - job: 'linux2020tool' + displayName: 'Linux Tool Run' + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy-test-proxy-docker.yml + - template : /eng/pipelines/templates/steps/check-availability.yml \ No newline at end of file diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 31e70001673b..e6f895b875ec 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -282,3 +282,8 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null) return $null } } + +function Import-Dev-Cert-python() +{ + Write-Host "Certificate action here!" +} From 6533a2be0bdfa0bba8455860d7637269f8662bc5 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 12 Aug 2021 17:29:45 -0700 Subject: [PATCH 55/90] update yaml names --- eng/pipelines/test-with-proxy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index dba95f0f67b3..807be8a05f1f 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -13,7 +13,7 @@ jobs: displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) - - template: /eng/common/testproxy-test-proxy-docker.yml + - template: /eng/common/test-proxy-docker.yml - template : /eng/pipelines/templates/steps/check-availability.yml - job: 'win2019tool' @@ -25,7 +25,7 @@ jobs: displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) - - template: /eng/common/testproxy-test-proxy-docker.yml + - template: /eng/common/test-proxy-tool.yml - template : /eng/pipelines/templates/steps/check-availability.yml - job: 'linux2020docker' @@ -37,7 +37,7 @@ jobs: displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) - - template: /eng/common/testproxy-test-proxy-docker.yml + - template: /eng/common/test-proxy-docker.yml - template : /eng/pipelines/templates/steps/check-availability.yml - job: 'linux2020tool' @@ -49,5 +49,5 @@ jobs: displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) - - template: /eng/common/testproxy-test-proxy-docker.yml + - template: /eng/common/test-proxy-tool.yml - template : /eng/pipelines/templates/steps/check-availability.yml \ No newline at end of file From fa6b9d724715b43e1f67496ee7e4947c962b6c60 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 12 Aug 2021 17:30:27 -0700 Subject: [PATCH 56/90] testproxy folder necessary --- eng/pipelines/test-with-proxy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 807be8a05f1f..d93941ea3629 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -13,7 +13,7 @@ jobs: displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) - - template: /eng/common/test-proxy-docker.yml + - template: /eng/common/testproxy/test-proxy-docker.yml - template : /eng/pipelines/templates/steps/check-availability.yml - job: 'win2019tool' @@ -25,7 +25,7 @@ jobs: displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) - - template: /eng/common/test-proxy-tool.yml + - template: /eng/common/testproxy/test-proxy-tool.yml - template : /eng/pipelines/templates/steps/check-availability.yml - job: 'linux2020docker' @@ -37,7 +37,7 @@ jobs: displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) - - template: /eng/common/test-proxy-docker.yml + - template: /eng/common/testproxy/test-proxy-docker.yml - template : /eng/pipelines/templates/steps/check-availability.yml - job: 'linux2020tool' @@ -49,5 +49,5 @@ jobs: displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) - - template: /eng/common/test-proxy-tool.yml + - template: /eng/common/testproxy/test-proxy-tool.yml - template : /eng/pipelines/templates/steps/check-availability.yml \ No newline at end of file From 2d34aeb1ce5bc5a5a36301c1b7eaa3600d5c9a69 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 12 Aug 2021 17:36:11 -0700 Subject: [PATCH 57/90] disable certificate trust --- eng/common/testproxy/test-proxy-docker.yml | 14 +++++++------- eng/common/testproxy/test-proxy-tool.yml | 14 +++++++------- eng/scripts/Language-Settings.ps1 | 3 +-- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index a86944ac83e8..b9371d68cc88 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -2,14 +2,14 @@ parameters: rootFolder: '$(Build.SourcesDirectory)' steps: - - pwsh: | - . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 + # - pwsh: | + # . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 - if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) - { - &$FindArtifactForApiReviewFn - } - displayName: 'Language Specific Certificate Trust' + # if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) + # { + # &$FindArtifactForApiReviewFn + # } + # displayName: 'Language Specific Certificate Trust' - pwsh: | $(Build.SourcesDirectory)/eng/common/scripts/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 78a6e2b8d7c7..4b268ea25360 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -2,14 +2,14 @@ parameters: rootFolder: '$(Build.SourcesDirectory)' steps: - - pwsh: | - . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 + # - pwsh: | + # . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 - if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) - { - &$FindArtifactForApiReviewFn - } - displayName: 'Language Specific Certificate Trust' + # if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) + # { + # &$FindArtifactForApiReviewFn + # } + # displayName: 'Language Specific Certificate Trust' - pwsh: | Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]$(Build.SourcesDirectory)/eng/common/testproxy/dotnet-devcert.pfx" diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index e6f895b875ec..2dcab179bdd9 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -282,8 +282,7 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null) return $null } } - -function Import-Dev-Cert-python() +function Import-Dev-Cert-python { Write-Host "Certificate action here!" } From cee2b4714b7a1d95d8d2b13801570c492dbc2cfc Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 12 Aug 2021 17:40:16 -0700 Subject: [PATCH 58/90] fix the docker issue --- eng/common/testproxy/test-proxy-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index b9371d68cc88..128904e8b0fb 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -12,5 +12,5 @@ steps: # displayName: 'Language Specific Certificate Trust' - pwsh: | - $(Build.SourcesDirectory)/eng/common/scripts/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" + $(Build.SourcesDirectory)/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" displayName: 'Run the docker container' From b028c9ca2d6f4128bef4222740a0111d1b2e0760 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 12 Aug 2021 17:44:57 -0700 Subject: [PATCH 59/90] use the appropriate .net version --- eng/common/testproxy/test-proxy-tool.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 4b268ea25360..0fc3792a4d1f 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -11,11 +11,21 @@ steps: # } # displayName: 'Language Specific Certificate Trust' + - pwsh: | + Write-Host "##vso[task.setvariable variable=OriginalPath]$env:PATH" + displayName: 'Store Path Value' + - pwsh: | Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]$(Build.SourcesDirectory)/eng/common/testproxy/dotnet-devcert.pfx" Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password" displayName: 'Configure Kestrel Environment Variables' + - task: UseDotNet@2 + displayName: "Use .NET Core SDK" + inputs: + packageType: sdk + version: 5.0.205 + - pwsh: | dotnet tool install azure.sdk.tools.testproxy ` --tool-path $(Build.BinariesDirectory)/test-proxy ` @@ -33,4 +43,8 @@ steps: nohup $(Build.BinariesDirectory)/test-proxy/test-proxy ` --storage-location "${{ parameters.rootFolder }}" & } - displayName: 'Run the testproxy via dotnet tool' \ No newline at end of file + displayName: 'Run the testproxy via dotnet tool' + + - pwsh: | + Write-Host "##vso[task.setvariable variable=PATH]$env:OriginalPath" + displayName: 'Restore .NET version by resetting path' \ No newline at end of file From 3ef56902fd73b7b2410244e1f4050ad222f98981 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 12 Aug 2021 17:47:42 -0700 Subject: [PATCH 60/90] include correct location --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 0fc3792a4d1f..aa8391d50f20 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -29,7 +29,7 @@ steps: - pwsh: | dotnet tool install azure.sdk.tools.testproxy ` --tool-path $(Build.BinariesDirectory)/test-proxy ` - --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json ` + --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 displayName: "Install test-proxy" From 04b2a0c7d59721789c701bd89b18fce74fea17a1 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 10:32:17 -0700 Subject: [PATCH 61/90] adjust to access the variable via devops variable --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index aa8391d50f20..7af3c84c569e 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -46,5 +46,5 @@ steps: displayName: 'Run the testproxy via dotnet tool' - pwsh: | - Write-Host "##vso[task.setvariable variable=PATH]$env:OriginalPath" + Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" displayName: 'Restore .NET version by resetting path' \ No newline at end of file From 4b1ac0f5772ecc19ca91bc09d27657576bff8ea4 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 10:52:34 -0700 Subject: [PATCH 62/90] list running containers --- eng/common/testproxy/test-proxy-docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index 128904e8b0fb..8331aee8ce8e 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -14,3 +14,7 @@ steps: - pwsh: | $(Build.SourcesDirectory)/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" displayName: 'Run the docker container' + + - pwsh: | + docker container ls -A + displayName: Check running container \ No newline at end of file From 7affd0f79eb1f941c35474b10becd5d109c0377b Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 10:54:49 -0700 Subject: [PATCH 63/90] adjust A to a --- eng/common/testproxy/test-proxy-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index 8331aee8ce8e..c010d4511b81 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -16,5 +16,5 @@ steps: displayName: 'Run the docker container' - pwsh: | - docker container ls -A + docker container ls -a displayName: Check running container \ No newline at end of file From 27b1d404fa3cb9f1e3375e4e6eb45e2ed61ac771 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:13:20 -0700 Subject: [PATCH 64/90] remove background run from nohup call --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 7af3c84c569e..84031af662aa 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -41,7 +41,7 @@ steps: } else { nohup $(Build.BinariesDirectory)/test-proxy/test-proxy ` - --storage-location "${{ parameters.rootFolder }}" & + --storage-location "${{ parameters.rootFolder }}" } displayName: 'Run the testproxy via dotnet tool' From aea92255bb042f274f53ba325c19c188d637087e Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:15:40 -0700 Subject: [PATCH 65/90] comment working tests --- eng/pipelines/test-with-proxy.yml | 66 +++++++++++++++---------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index d93941ea3629..3ce2ba3c98c2 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -4,41 +4,41 @@ variables: PythonVersion: '3.9' jobs: - - job: 'win2019docker' - displayName: 'Windows Docker Run' - pool: - vmImage: 'windows-2019' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' - inputs: - versionSpec: $(PythonVersion) - - template: /eng/common/testproxy/test-proxy-docker.yml - - template : /eng/pipelines/templates/steps/check-availability.yml + # - job: 'win2019docker' + # displayName: 'Windows Docker Run' + # pool: + # vmImage: 'windows-2019' + # steps: + # - task: UsePythonVersion@0 + # displayName: 'Use Python $(PythonVersion)' + # inputs: + # versionSpec: $(PythonVersion) + # - template: /eng/common/testproxy/test-proxy-docker.yml + # - template : /eng/pipelines/templates/steps/check-availability.yml - - job: 'win2019tool' - displayName: 'Windows Tool Run' - pool: - vmImage: 'windows-2019' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' - inputs: - versionSpec: $(PythonVersion) - - template: /eng/common/testproxy/test-proxy-tool.yml - - template : /eng/pipelines/templates/steps/check-availability.yml + # - job: 'win2019tool' + # displayName: 'Windows Tool Run' + # pool: + # vmImage: 'windows-2019' + # steps: + # - task: UsePythonVersion@0 + # displayName: 'Use Python $(PythonVersion)' + # inputs: + # versionSpec: $(PythonVersion) + # - template: /eng/common/testproxy/test-proxy-tool.yml + # - template : /eng/pipelines/templates/steps/check-availability.yml - - job: 'linux2020docker' - displayName: 'Linux Docker Run' - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' - inputs: - versionSpec: $(PythonVersion) - - template: /eng/common/testproxy/test-proxy-docker.yml - - template : /eng/pipelines/templates/steps/check-availability.yml + # - job: 'linux2020docker' + # displayName: 'Linux Docker Run' + # pool: + # vmImage: 'ubuntu-20.04' + # steps: + # - task: UsePythonVersion@0 + # displayName: 'Use Python $(PythonVersion)' + # inputs: + # versionSpec: $(PythonVersion) + # - template: /eng/common/testproxy/test-proxy-docker.yml + # - template : /eng/pipelines/templates/steps/check-availability.yml - job: 'linux2020tool' displayName: 'Linux Tool Run' From 1e447a55353e0d13ef7eaea6ea23c637a225d536 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:28:29 -0700 Subject: [PATCH 66/90] additional debugging --- eng/common/testproxy/test-proxy-tool.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 84031af662aa..0a135e0d6ec3 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -41,10 +41,17 @@ steps: } else { nohup $(Build.BinariesDirectory)/test-proxy/test-proxy ` - --storage-location "${{ parameters.rootFolder }}" + --storage-location "${{ parameters.rootFolder }}" & } + displayName: 'Run the testproxy via dotnet tool' - + + - pwsh: | + cat nohup.txt + displayName: 'Check Progress' + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + + - pwsh: | Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" displayName: 'Restore .NET version by resetting path' \ No newline at end of file From 48389212df9ba080b08f8b0c40cca67882f87a43 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:34:06 -0700 Subject: [PATCH 67/90] nohup.txt isn't getting any text --- eng/common/testproxy/test-proxy-tool.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 0a135e0d6ec3..9646e477f3c8 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -40,8 +40,9 @@ steps: -NoNewWindow -PassThru } else { - nohup $(Build.BinariesDirectory)/test-proxy/test-proxy ` - --storage-location "${{ parameters.rootFolder }}" & + Write-Host ("Running in " + Get-Location) + Write-Host "nohup $(Build.BinariesDirectory)/test-proxy/test-proxy --storage-location '${{ parameters.rootFolder }}' &" + nohup $(Build.BinariesDirectory)/test-proxy/test-proxy --storage-location "${{ parameters.rootFolder }}" & } displayName: 'Run the testproxy via dotnet tool' From 8d57654fcf5e4566196c1494124b2831d5fd6892 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:35:57 -0700 Subject: [PATCH 68/90] remove flavortext --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 9646e477f3c8..3ae5f9edd995 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -40,7 +40,7 @@ steps: -NoNewWindow -PassThru } else { - Write-Host ("Running in " + Get-Location) + Write-Host Get-Location Write-Host "nohup $(Build.BinariesDirectory)/test-proxy/test-proxy --storage-location '${{ parameters.rootFolder }}' &" nohup $(Build.BinariesDirectory)/test-proxy/test-proxy --storage-location "${{ parameters.rootFolder }}" & } From d8f27c32c3474a62d3d5ccc44a49ece91d89cf72 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:37:09 -0700 Subject: [PATCH 69/90] its nohup.out. not nohup --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 3ae5f9edd995..2b687632283f 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -48,7 +48,7 @@ steps: displayName: 'Run the testproxy via dotnet tool' - pwsh: | - cat nohup.txt + cat nohup.out displayName: 'Check Progress' condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) From be392c226f2d044b98a32eeed0155146497ed8b7 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 13:34:19 -0700 Subject: [PATCH 70/90] attempting to use shell --- eng/common/testproxy/invoke-proxy.sh | 1 + eng/common/testproxy/test-proxy-tool.yml | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 eng/common/testproxy/invoke-proxy.sh diff --git a/eng/common/testproxy/invoke-proxy.sh b/eng/common/testproxy/invoke-proxy.sh new file mode 100644 index 000000000000..be71a0e69212 --- /dev/null +++ b/eng/common/testproxy/invoke-proxy.sh @@ -0,0 +1 @@ +${TEST_PROXY_LOC}/test-proxy \ No newline at end of file diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 2b687632283f..e66d62a278f3 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -27,10 +27,14 @@ steps: version: 5.0.205 - pwsh: | + $toolLocation = "$(Build.BinariesDirectory)/test-proxy" + dotnet tool install azure.sdk.tools.testproxy ` - --tool-path $(Build.BinariesDirectory)/test-proxy ` + --tool-path $toolLocation` --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 + + Write-Host "##vso[task.setvariable variable=TEST_PROXY_LOC]$toolLocation" displayName: "Install test-proxy" - pwsh: | @@ -40,12 +44,11 @@ steps: -NoNewWindow -PassThru } else { - Write-Host Get-Location - Write-Host "nohup $(Build.BinariesDirectory)/test-proxy/test-proxy --storage-location '${{ parameters.rootFolder }}' &" - nohup $(Build.BinariesDirectory)/test-proxy/test-proxy --storage-location "${{ parameters.rootFolder }}" & + Get-Location + nohup $(Build.SourcesDirectory)/eng/common/testproxy/invoke-proxy.sh & } - displayName: 'Run the testproxy via dotnet tool' + workingDirectory: "${{ parameters.rootFolder }}" - pwsh: | cat nohup.out From e9995f59f6b4f579abc6f2b5461d3fbd775dcca0 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 13:38:20 -0700 Subject: [PATCH 71/90] remove progress check --- eng/common/testproxy/test-proxy-tool.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index e66d62a278f3..bf9e4443df40 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -50,12 +50,6 @@ steps: displayName: 'Run the testproxy via dotnet tool' workingDirectory: "${{ parameters.rootFolder }}" - - pwsh: | - cat nohup.out - displayName: 'Check Progress' - condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - - - pwsh: | Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" displayName: 'Restore .NET version by resetting path' \ No newline at end of file From 8f1d1f64b6cc50f48a1d38eafceb7478a2539de1 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 13:52:30 -0700 Subject: [PATCH 72/90] call nohup from bash --- eng/common/testproxy/test-proxy-tool.yml | 37 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index bf9e4443df40..0a63eaf34806 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -38,18 +38,37 @@ steps: displayName: "Install test-proxy" - pwsh: | - if ($IsWindows) { - Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` - -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` - -NoNewWindow -PassThru - } - else { - Get-Location - nohup $(Build.SourcesDirectory)/eng/common/testproxy/invoke-proxy.sh & - } + Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` + -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` + -NoNewWindow -PassThru displayName: 'Run the testproxy via dotnet tool' + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) + + # - pwsh: | + # if ($IsWindows) { + # Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` + # -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` + # -NoNewWindow -PassThru + # } + # else { + # Get-Location + # nohup $(Build.SourcesDirectory)/eng/common/testproxy/invoke-proxy.sh & + # } + # displayName: 'Run the testproxy via dotnet tool' + # workingDirectory: "${{ parameters.rootFolder }}" + + - bash: | + nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & + displayName: "Run the testproxy via dotnet tool" + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" + # - pwsh: | + # cat nohup.out + # displayName: 'Check Progress' + # condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + + - pwsh: | Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" displayName: 'Restore .NET version by resetting path' \ No newline at end of file From 425608d6a4079013fdd0c3e991e3f87ae06dd822 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 13:56:51 -0700 Subject: [PATCH 73/90] dump output --- eng/common/testproxy/test-proxy-tool.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 0a63eaf34806..35db190e3891 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -34,7 +34,8 @@ steps: --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 - Write-Host "##vso[task.setvariable variable=TEST_PROXY_LOC]$toolLocation" + Get-ChildItem -R $(Build.BinariesDirectory)/test-proxy + # Write-Host "##vso[task.setvariable variable=TEST_PROXY_LOC]$toolLocation" displayName: "Install test-proxy" - pwsh: | From c5d226514415d1fc517d521d6fcba7e3ff2b93ea Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:00:25 -0700 Subject: [PATCH 74/90] attempt another shot at this --- eng/common/testproxy/test-proxy-tool.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 35db190e3891..cb5ebba6bcce 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -27,14 +27,12 @@ steps: version: 5.0.205 - pwsh: | - $toolLocation = "$(Build.BinariesDirectory)/test-proxy" - dotnet tool install azure.sdk.tools.testproxy ` - --tool-path $toolLocation` + --tool-path $(Build.BinariesDirectory)/test-proxy` --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 - Get-ChildItem -R $(Build.BinariesDirectory)/test-proxy + Get-ChildItem -R $(Build.BinariesDirectory) # Write-Host "##vso[task.setvariable variable=TEST_PROXY_LOC]$toolLocation" displayName: "Install test-proxy" @@ -59,7 +57,7 @@ steps: # workingDirectory: "${{ parameters.rootFolder }}" - bash: | - nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & + nohup test-proxy & displayName: "Run the testproxy via dotnet tool" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" From 40f502b311708857211eb9e85a8ca8d9a5a76388 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:04:02 -0700 Subject: [PATCH 75/90] why ending early? --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index cb5ebba6bcce..ff9d63b3671f 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -32,7 +32,7 @@ steps: --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 - Get-ChildItem -R $(Build.BinariesDirectory) + Get-ChildItem -R $(Build.BinariesDirectory)/test-proxy/ # Write-Host "##vso[task.setvariable variable=TEST_PROXY_LOC]$toolLocation" displayName: "Install test-proxy" From 6c350f52ec93d2ee01438cd053cff1ac9b15fb4a Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:06:32 -0700 Subject: [PATCH 76/90] surround in quotes? --- eng/common/testproxy/test-proxy-tool.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index ff9d63b3671f..4b02ef264ee8 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -32,8 +32,7 @@ steps: --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 - Get-ChildItem -R $(Build.BinariesDirectory)/test-proxy/ - # Write-Host "##vso[task.setvariable variable=TEST_PROXY_LOC]$toolLocation" + Get-ChildItem -R -Path "$(Build.BinariesDirectory)/test-proxy/" displayName: "Install test-proxy" - pwsh: | From 4bca47944b95fa8b4fde34028122b35f82179f93 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:17:38 -0700 Subject: [PATCH 77/90] splitting win/linux invocations and installs entirely --- eng/common/testproxy/test-proxy-tool.yml | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 4b02ef264ee8..a4d6fbe1fc27 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -31,15 +31,13 @@ steps: --tool-path $(Build.BinariesDirectory)/test-proxy` --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 - - Get-ChildItem -R -Path "$(Build.BinariesDirectory)/test-proxy/" - displayName: "Install test-proxy" + displayName: "Install test-proxy - windows" - pwsh: | Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` -NoNewWindow -PassThru - displayName: 'Run the testproxy via dotnet tool' + displayName: 'Run the testproxy via dotnet tool - windows' condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) # - pwsh: | @@ -55,18 +53,21 @@ steps: # displayName: 'Run the testproxy via dotnet tool' # workingDirectory: "${{ parameters.rootFolder }}" + + - pwsh: | + dotnet tool install azure.sdk.tools.testproxy ` + --global ` + --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` + --version 1.0.0-dev.20210811.2 + displayName: "Install test-proxy - linux/mac" + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + - bash: | - nohup test-proxy & - displayName: "Run the testproxy via dotnet tool" + nohup /root/.dotnet/tools/test-proxy & + displayName: "Run the testproxy - linux/mac" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" - # - pwsh: | - # cat nohup.out - # displayName: 'Check Progress' - # condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - - - pwsh: | Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" displayName: 'Restore .NET version by resetting path' \ No newline at end of file From 32fa8819af86499bf2ae06726f4db86d3bd307a6 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:20:52 -0700 Subject: [PATCH 78/90] maybe? --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index a4d6fbe1fc27..73d738c9a911 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -63,7 +63,7 @@ steps: condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - bash: | - nohup /root/.dotnet/tools/test-proxy & + sudo nohup /root/.dotnet/tools/test-proxy & displayName: "Run the testproxy - linux/mac" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" From aa79780dbc5149b830d6aaca4b928bf291870adc Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:26:03 -0700 Subject: [PATCH 79/90] force install and run right after each other --- eng/common/testproxy/test-proxy-tool.yml | 15 +++--- eng/pipelines/test-with-proxy.yml | 66 ++++++++++++------------ 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 73d738c9a911..d11e2c90e9f4 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -32,6 +32,7 @@ steps: --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 displayName: "Install test-proxy - windows" + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) - pwsh: | Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` @@ -59,14 +60,16 @@ steps: --global ` --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 - displayName: "Install test-proxy - linux/mac" - condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - - bash: | - sudo nohup /root/.dotnet/tools/test-proxy & - displayName: "Run the testproxy - linux/mac" + sudo nohup test-proxy & + displayName: "Install and run test-proxy - linux/mac" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - workingDirectory: "${{ parameters.rootFolder }}" + + # - bash: | + + # displayName: "Run the testproxy - linux/mac" + # condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + # workingDirectory: "${{ parameters.rootFolder }}" - pwsh: | Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml index 3ce2ba3c98c2..d93941ea3629 100644 --- a/eng/pipelines/test-with-proxy.yml +++ b/eng/pipelines/test-with-proxy.yml @@ -4,41 +4,41 @@ variables: PythonVersion: '3.9' jobs: - # - job: 'win2019docker' - # displayName: 'Windows Docker Run' - # pool: - # vmImage: 'windows-2019' - # steps: - # - task: UsePythonVersion@0 - # displayName: 'Use Python $(PythonVersion)' - # inputs: - # versionSpec: $(PythonVersion) - # - template: /eng/common/testproxy/test-proxy-docker.yml - # - template : /eng/pipelines/templates/steps/check-availability.yml + - job: 'win2019docker' + displayName: 'Windows Docker Run' + pool: + vmImage: 'windows-2019' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy/test-proxy-docker.yml + - template : /eng/pipelines/templates/steps/check-availability.yml - # - job: 'win2019tool' - # displayName: 'Windows Tool Run' - # pool: - # vmImage: 'windows-2019' - # steps: - # - task: UsePythonVersion@0 - # displayName: 'Use Python $(PythonVersion)' - # inputs: - # versionSpec: $(PythonVersion) - # - template: /eng/common/testproxy/test-proxy-tool.yml - # - template : /eng/pipelines/templates/steps/check-availability.yml + - job: 'win2019tool' + displayName: 'Windows Tool Run' + pool: + vmImage: 'windows-2019' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy/test-proxy-tool.yml + - template : /eng/pipelines/templates/steps/check-availability.yml - # - job: 'linux2020docker' - # displayName: 'Linux Docker Run' - # pool: - # vmImage: 'ubuntu-20.04' - # steps: - # - task: UsePythonVersion@0 - # displayName: 'Use Python $(PythonVersion)' - # inputs: - # versionSpec: $(PythonVersion) - # - template: /eng/common/testproxy/test-proxy-docker.yml - # - template : /eng/pipelines/templates/steps/check-availability.yml + - job: 'linux2020docker' + displayName: 'Linux Docker Run' + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy/test-proxy-docker.yml + - template : /eng/pipelines/templates/steps/check-availability.yml - job: 'linux2020tool' displayName: 'Linux Tool Run' From 72c76cc4cd6a98a370299434cbfc5922c674dcc1 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:58:26 -0700 Subject: [PATCH 80/90] another attempt at this --- eng/common/testproxy/test-proxy-tool.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index d11e2c90e9f4..bba8cf86852b 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -28,7 +28,7 @@ steps: - pwsh: | dotnet tool install azure.sdk.tools.testproxy ` - --tool-path $(Build.BinariesDirectory)/test-proxy` + --tool-path $(Build.BinariesDirectory)/test-proxy ` --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 displayName: "Install test-proxy - windows" @@ -62,14 +62,15 @@ steps: --version 1.0.0-dev.20210811.2 sudo nohup test-proxy & + Get-Process displayName: "Install and run test-proxy - linux/mac" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - # - bash: | - - # displayName: "Run the testproxy - linux/mac" - # condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - # workingDirectory: "${{ parameters.rootFolder }}" + - bash: | + sudo ps -a + displayName: "Check Running Processes" + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + workingDirectory: "${{ parameters.rootFolder }}" - pwsh: | Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" From 40b6787dae6c1c017f7500ed4c451d6ece31ce18 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:08:53 -0700 Subject: [PATCH 81/90] adjust installation back to path --- eng/common/testproxy/test-proxy-tool.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index bba8cf86852b..83b5cb67d3fd 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -57,17 +57,14 @@ steps: - pwsh: | dotnet tool install azure.sdk.tools.testproxy ` - --global ` + --tool-path $(Build.BinariesDirectory)/test-proxy ` --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 - - sudo nohup test-proxy & - Get-Process displayName: "Install and run test-proxy - linux/mac" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - bash: | - sudo ps -a + nohup $(Build.BinariesDirectory)/test-proxy & displayName: "Check Running Processes" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" From f3c84d9abf9e03ce72b9431d77d8fa4b83cfc7cd Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:11:52 -0700 Subject: [PATCH 82/90] sudo the nohup! --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 83b5cb67d3fd..b30ba5348a03 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -64,7 +64,7 @@ steps: condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - bash: | - nohup $(Build.BinariesDirectory)/test-proxy & + sudo nohup $(Build.BinariesDirectory)/test-proxy & displayName: "Check Running Processes" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" From aba2c9bcb7190fa77146abe8f3510e6eff7a0212 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:20:42 -0700 Subject: [PATCH 83/90] test-proxy --- eng/common/testproxy/test-proxy-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index b30ba5348a03..40a95d847da8 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -64,7 +64,7 @@ steps: condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - bash: | - sudo nohup $(Build.BinariesDirectory)/test-proxy & + sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & displayName: "Check Running Processes" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" From d9931d90715658ee234bd3f2f9214e735ad8519d Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:27:28 -0700 Subject: [PATCH 84/90] unify the install again --- eng/common/testproxy/test-proxy-tool.yml | 47 ++++++------------- .../templates/steps/check-availability.yml | 6 +-- 2 files changed, 16 insertions(+), 37 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 40a95d847da8..45352ac5258b 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -32,42 +32,25 @@ steps: --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 displayName: "Install test-proxy - windows" - condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) - pwsh: | - Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` - -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` - -NoNewWindow -PassThru - displayName: 'Run the testproxy via dotnet tool - windows' - condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) - - # - pwsh: | - # if ($IsWindows) { - # Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` - # -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` - # -NoNewWindow -PassThru - # } - # else { - # Get-Location - # nohup $(Build.SourcesDirectory)/eng/common/testproxy/invoke-proxy.sh & - # } - # displayName: 'Run the testproxy via dotnet tool' - # workingDirectory: "${{ parameters.rootFolder }}" - + if ($IsWindows) { + Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` + -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` + -NoNewWindow -PassThru + } + else { + nohup $(Build.SourcesDirectory)/test-proxy/test-proxy & + } + displayName: 'Run the testproxy via dotnet tool' + workingDirectory: "${{ parameters.rootFolder }}" - - pwsh: | - dotnet tool install azure.sdk.tools.testproxy ` - --tool-path $(Build.BinariesDirectory)/test-proxy ` - --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` - --version 1.0.0-dev.20210811.2 - displayName: "Install and run test-proxy - linux/mac" - condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - - bash: | - sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & - displayName: "Check Running Processes" - condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - workingDirectory: "${{ parameters.rootFolder }}" + # - bash: | + # sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & + # displayName: "Check Running Processes" + # condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + # workingDirectory: "${{ parameters.rootFolder }}" - pwsh: | Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" diff --git a/eng/pipelines/templates/steps/check-availability.yml b/eng/pipelines/templates/steps/check-availability.yml index 01d24a283739..7e9fb753220e 100644 --- a/eng/pipelines/templates/steps/check-availability.yml +++ b/eng/pipelines/templates/steps/check-availability.yml @@ -1,8 +1,4 @@ steps: - pwsh: | Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" - displayName: 'http://localhost:5000/Info/Available' - - - pwsh: | - Invoke-WebRequest -Uri "https://localhost:5001/Info/Available" -SkipCertificateCheck - displayName: 'https://localhost:5001/Info/Available' \ No newline at end of file + displayName: 'http://localhost:5000/Info/Available' \ No newline at end of file From ab08e24178c77975127b5d19f84fe301349ddadc Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:34:24 -0700 Subject: [PATCH 85/90] linux tool running? --- eng/common/testproxy/test-proxy-tool.yml | 26 +++++++++--------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 45352ac5258b..64f1088f9f8f 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -34,24 +34,18 @@ steps: displayName: "Install test-proxy - windows" - pwsh: | - if ($IsWindows) { - Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` - -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` - -NoNewWindow -PassThru - } - else { - nohup $(Build.SourcesDirectory)/test-proxy/test-proxy & - } - displayName: 'Run the testproxy via dotnet tool' + Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` + -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` + -NoNewWindow -PassThru + displayName: 'Run the testproxy via dotnet tool - windows' + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) + + - bash: | + sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & + displayName: "Check Running Processes" + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" - - # - bash: | - # sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & - # displayName: "Check Running Processes" - # condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) - # workingDirectory: "${{ parameters.rootFolder }}" - - pwsh: | Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" displayName: 'Restore .NET version by resetting path' \ No newline at end of file From 6129d38d0aca5f60bd3101c962eaa9597371cbdd Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:36:19 -0700 Subject: [PATCH 86/90] unify --- eng/common/testproxy/test-proxy-tool.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 64f1088f9f8f..5cfcc82eb3ab 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -31,18 +31,18 @@ steps: --tool-path $(Build.BinariesDirectory)/test-proxy ` --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version 1.0.0-dev.20210811.2 - displayName: "Install test-proxy - windows" + displayName: "Install test-proxy" - pwsh: | Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` -NoNewWindow -PassThru - displayName: 'Run the testproxy via dotnet tool - windows' + displayName: 'Run the testproxy - windows' condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) - bash: | sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & - displayName: "Check Running Processes" + displayName: "Run the testproxy - linux/mac" condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) workingDirectory: "${{ parameters.rootFolder }}" From 9a970abbba51babe2a976a0b6b66da526d85797b Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:49:52 -0700 Subject: [PATCH 87/90] update to include --- eng/common/testproxy/test-proxy-docker.yml | 14 +++++++------- eng/common/testproxy/test-proxy-tool.yml | 15 ++++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index c010d4511b81..e7105e43a680 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -2,14 +2,14 @@ parameters: rootFolder: '$(Build.SourcesDirectory)' steps: - # - pwsh: | - # . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 + - pwsh: | + . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 - # if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) - # { - # &$FindArtifactForApiReviewFn - # } - # displayName: 'Language Specific Certificate Trust' + if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) + { + &$FindArtifactForApiReviewFn + } + displayName: 'Language Specific Certificate Trust' - pwsh: | $(Build.SourcesDirectory)/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 5cfcc82eb3ab..c31641448bfd 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -2,14 +2,14 @@ parameters: rootFolder: '$(Build.SourcesDirectory)' steps: - # - pwsh: | - # . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 + - pwsh: | + . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 - # if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) - # { - # &$FindArtifactForApiReviewFn - # } - # displayName: 'Language Specific Certificate Trust' + if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) + { + &$FindArtifactForApiReviewFn + } + displayName: 'Language Specific Certificate Trust' - pwsh: | Write-Host "##vso[task.setvariable variable=OriginalPath]$env:PATH" @@ -40,6 +40,7 @@ steps: displayName: 'Run the testproxy - windows' condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) + # nohup does NOT continue beyond the current session if you use it within powershell - bash: | sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & displayName: "Run the testproxy - linux/mac" From c2ccb07d8f74e835bc11fc2f665c7de220e0cd1c Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 16 Aug 2021 11:00:02 -0700 Subject: [PATCH 88/90] update to call the correct function --- eng/common/testproxy/test-proxy-docker.yml | 4 ++-- eng/common/testproxy/test-proxy-tool.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index e7105e43a680..08bd93cf10cf 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -5,9 +5,9 @@ steps: - pwsh: | . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 - if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) + if ($TestProxyTrustCertFn -and (Test-Path "Function:$TestProxyTrustCertFn")) { - &$FindArtifactForApiReviewFn + &$TestProxyTrustCertFn } displayName: 'Language Specific Certificate Trust' diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index c31641448bfd..5a803021a78a 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -5,9 +5,9 @@ steps: - pwsh: | . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 - if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) + if ($TestProxyTrustCertFn -and (Test-Path "Function:$TestProxyTrustCertFn")) { - &$FindArtifactForApiReviewFn + &$TestProxyTrustCertFn } displayName: 'Language Specific Certificate Trust' From 52b0138a37c073ecf42c505d5bb1688f570c02b4 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 16 Aug 2021 12:35:23 -0700 Subject: [PATCH 89/90] swap to using the certificate --- eng/common/scripts/trust-proxy-certificate.ps1 | 6 ++++++ eng/common/testproxy/test-proxy-docker.yml | 7 +------ eng/common/testproxy/test-proxy-tool.yml | 7 +------ 3 files changed, 8 insertions(+), 12 deletions(-) create mode 100644 eng/common/scripts/trust-proxy-certificate.ps1 diff --git a/eng/common/scripts/trust-proxy-certificate.ps1 b/eng/common/scripts/trust-proxy-certificate.ps1 new file mode 100644 index 000000000000..144d304cfd18 --- /dev/null +++ b/eng/common/scripts/trust-proxy-certificate.ps1 @@ -0,0 +1,6 @@ +. $PSScriptRoot/common.ps1 + +if ($TestProxyTrustCertFn -and (Test-Path "Function:$TestProxyTrustCertFn")) +{ + &$TestProxyTrustCertFn +} \ No newline at end of file diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index 08bd93cf10cf..97617b6fd08a 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -3,12 +3,7 @@ parameters: steps: - pwsh: | - . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 - - if ($TestProxyTrustCertFn -and (Test-Path "Function:$TestProxyTrustCertFn")) - { - &$TestProxyTrustCertFn - } + $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1 displayName: 'Language Specific Certificate Trust' - pwsh: | diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 5a803021a78a..9f24b0f0d527 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -3,12 +3,7 @@ parameters: steps: - pwsh: | - . $(Build.SourcesDirectory)/eng/common/scripts/common.ps1 - - if ($TestProxyTrustCertFn -and (Test-Path "Function:$TestProxyTrustCertFn")) - { - &$TestProxyTrustCertFn - } + $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1 displayName: 'Language Specific Certificate Trust' - pwsh: | From 830f3cd1d166caeb11b28edc09db8f12f242a41a Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 16 Aug 2021 17:14:44 -0700 Subject: [PATCH 90/90] updates for ascending the tree --- .../devtools_testutils/proxy_testcase.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 44667c1bf165..c2de8b0441ec 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------- import os import requests +import pdb try: # py3 @@ -37,11 +38,17 @@ def get_test_id(): # pytest sets the current running test in an environment variable - path_to_test = os.getenv("PYTEST_CURRENT_TEST").split(" ")[0].replace("::", ".") - beginning, end = path_to_test.split("/tests/") - full = beginning + "/tests/recordings/" + end - print(full) - return full + setting_value = os.getenv("PYTEST_CURRENT_TEST") + + path_to_test = os.path.normpath(setting_value.split(" ")[0]) + path_components = path_to_test.split(os.sep) + + for idx, val in enumerate(path_components): + if val.startswith("test"): + path_components.insert(idx + 1, "recordings") + break + + return os.sep.join(path_components).replace("::", "").replace("\\", "/") def get_current_sha():