diff --git a/doc/dev/test_proxy_migration_guide.md b/doc/dev/test_proxy_migration_guide.md index 51c50a1cae1c..d33d27911096 100644 --- a/doc/dev/test_proxy_migration_guide.md +++ b/doc/dev/test_proxy_migration_guide.md @@ -28,25 +28,25 @@ class TestExample(AzureTestCase): ### New test structure To use the proxy, test classes should inherit from AzureRecordedTestCase and recorded test methods should use a -RecordedByProxy decorator: +`recorded_by_proxy` decorator: ```py -from devtools_testutils import AzureRecordedTestCase, RecordedByProxy +from devtools_testutils import AzureRecordedTestCase, recorded_by_proxy class TestExample(AzureRecordedTestCase): - @RecordedByProxy + @recorded_by_proxy def test_example(self): ... @ExamplePreparer() - @RecordedByProxy + @recorded_by_proxy def test_example_with_preparer(self): ... ``` -For async tests, import the RecordedByProxyAsync decorator from `devtools_testutils.aio` and use it in the same -way as RecordedByProxy. +For async tests, import the `recorded_by_proxy_async` decorator from `devtools_testutils.aio` and use it in the same +way as `recorded_by_proxy`. > **Note:** since AzureRecordedTestCase doesn't inherit from `unittest.TestCase`, test class names need to start > with "Test" in order to be properly collected by pytest by default. For more information, please refer to @@ -135,6 +135,43 @@ made to `https://fakeendpoint-secondary.table.core.windows.net`, and URIs will a For more details about sanitizers and their options, please refer to [devtools_testutils/sanitizers.py][py_sanitizers]. +### Record test variables + +To run recorded tests successfully when there's an element of non-secret randomness to them, the test proxy provides a +[`variables` API](https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy#storing-variables). +This makes it possible for a test to record the values of variables that were used during recording and use the same +values in playback mode without a sanitizer. + +For example, imagine that a test uses a randomized `table_name` variable when creating resources. The same random value +for `table_name` can be used in playback mode by using this `variables` API. + +There are two requirements for a test to use recorded variables. First, the test method should accept `**kwargs` and/or +a `variables` parameter. Second, the test method should `return` a dictionary with any test variables that it wants to +record. This dictionary will be stored in the recording when the test is run live, and will be passed to the test as a +`variables` keyword argument when the test is run in playback. + +Below is a code example of how a test method could use recorded variables: + +```python +from devtools_testutils import AzureRecordedTestCase, recorded_by_proxy + +class TestExample(AzureRecordedTestCase): + + @recorded_by_proxy + def test_example(self, variables): + # in live mode, variables is an empty dictionary + # in playback mode, the value of variables is {"table_name": "random-value"} + if self.is_live: + table_name = "random-value" + variables = {"table_name": table_name} + + # use variables["table_name"] when using the table name throughout the test + ... + + # return the variables at the end of the test + return variables +``` + ## Implementation details ### What does the test proxy do? @@ -147,7 +184,7 @@ For example, if an operation would typically make a GET request to `https://localhost:5001/Tables` instead. The original endpoint should be stored in an `x-recording-upstream-base-uri` -- the proxy will send the original request and record the result. -The RecordedByProxy and RecordedByProxyAsync decorators patch test requests to do this for you. +The `recorded_by_proxy` and `recorded_by_proxy_async` decorators patch test requests to do this for you. ### How does the test proxy know when and what to record or play back? @@ -179,7 +216,7 @@ Running tests in playback follows the same pattern, except that requests will be `/playback/stop` instead. A header, `x-recording-mode`, should be set to `record` for all requests when recording and `playback` when playing recordings back. More details can be found [here][detailed_docs]. -The RecordedByProxy and RecordedByProxyAsync decorators send the appropriate requests at the start and end of each test +The `recorded_by_proxy` and `recorded_by_proxy_async` decorators send the appropriate requests at the start and end of each test case. [detailed_docs]: https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md diff --git a/tools/azure-sdk-tools/devtools_testutils/__init__.py b/tools/azure-sdk-tools/devtools_testutils/__init__.py index bcd0298d0c3c..67535183ba62 100644 --- a/tools/azure-sdk-tools/devtools_testutils/__init__.py +++ b/tools/azure-sdk-tools/devtools_testutils/__init__.py @@ -15,7 +15,7 @@ ) from .keyvault_preparer import KeyVaultPreparer from .powershell_preparer import PowerShellPreparer -from .proxy_testcase import RecordedByProxy +from .proxy_testcase import recorded_by_proxy from .sanitizers import ( add_body_key_sanitizer, add_body_regex_sanitizer, @@ -56,7 +56,7 @@ "RandomNameResourceGroupPreparer", "CachedResourceGroupPreparer", "PowerShellPreparer", - "RecordedByProxy", + "recorded_by_proxy", "ResponseCallback", "RetryCounter", "FakeTokenCredential", diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py b/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py index 5265d80fe58e..678c2dd89152 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py @@ -1,3 +1,3 @@ -from .proxy_testcase_async import RecordedByProxyAsync +from .proxy_testcase_async import recorded_by_proxy_async -__all__ = ["RecordedByProxyAsync"] +__all__ = ["recorded_by_proxy_async"] diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py index 11c8f9e44395..60e94c2812e6 100644 --- a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -3,6 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +import logging + from azure.core.pipeline.transport import AioHttpTransport from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function @@ -14,10 +16,16 @@ ) -def RecordedByProxyAsync(func): +def recorded_by_proxy_async(test_func): + """Decorator that redirects network requests to target the azure-sdk-tools test proxy. Use with recorded tests. + + For more details and usage examples, refer to + https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/test_proxy_migration_guide.md + """ + async def record_wrap(*args, **kwargs): test_id = get_test_id() - recording_id = start_record_or_playback(test_id) + recording_id, variables = start_record_or_playback(test_id) def transform_args(*args, **kwargs): copied_positional_args = list(args) @@ -28,23 +36,32 @@ def transform_args(*args, **kwargs): return tuple(copied_positional_args), kwargs trimmed_kwargs = {k: v for k, v in kwargs.items()} - trim_kwargs_from_test_function(func, trimmed_kwargs) + trim_kwargs_from_test_function(test_func, trimmed_kwargs) - original_func = AioHttpTransport.send + original_transport_func = AioHttpTransport.send async def combined_call(*args, **kwargs): adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) - return await original_func(*adjusted_args, **adjusted_kwargs) + return await original_transport_func(*adjusted_args, **adjusted_kwargs) AioHttpTransport.send = combined_call - # call the modified function. + # call the modified function + # we define test_output before invoking the test so the variable is defined in case of an exception + test_output = None try: - value = await func(*args, **trimmed_kwargs) + test_output = await test_func(*args, variables=variables, **trimmed_kwargs) + except TypeError: + logger = logging.getLogger() + logger.info( + "This test can't accept variables as input. The test method should accept `**kwargs` and/or a " + "`variables` parameter to make use of recorded test variables." + ) + test_output = await test_func(*args, **trimmed_kwargs) finally: - AioHttpTransport.send = original_func - stop_record_or_playback(test_id, recording_id) + AioHttpTransport.send = original_transport_func + stop_record_or_playback(test_id, recording_id, test_output) - return value + return test_output return record_wrap diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 78918ab0063b..a189658aaea8 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -4,7 +4,10 @@ # license information. # -------------------------------------------------------------------------- import os +import logging import requests +import six +from typing import TYPE_CHECKING try: # py3 @@ -23,6 +26,9 @@ from .azure_recorded_testcase import is_live from .config import PROXY_URL +if TYPE_CHECKING: + from typing import Tuple + # To learn about how to migrate SDK tests to the test proxy, please refer to the migration guide at # https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/test_proxy_migration_guide.md @@ -40,6 +46,7 @@ def get_test_id(): + # type: () -> str # pytest sets the current running test in an environment variable setting_value = os.getenv("PYTEST_CURRENT_TEST") @@ -55,8 +62,15 @@ def get_test_id(): def start_record_or_playback(test_id): - result = subprocess.check_output(["git", "rev-parse", "HEAD"]) - current_sha = result.decode("utf-8").strip() + # type: (str) -> Tuple(str, dict) + """Sends a request to begin recording or playing back the provided test. + + This returns a tuple, (a, b), where a is the recording ID of the test and b is the `variables` dictionary that maps + test variables to values. If no variable dictionary was stored when the test was recorded, b is an empty dictionary. + """ + head_commit = subprocess.check_output(["git", "rev-parse", "HEAD"]) + current_sha = head_commit.decode("utf-8").strip() + variables = {} # this stores a dictionary of test variable values that could have been stored with a recording if is_live(): result = requests.post( @@ -70,14 +84,28 @@ def start_record_or_playback(test_id): headers={"x-recording-file": test_id, "x-recording-sha": current_sha}, ) recording_id = result.headers["x-recording-id"] - return recording_id + try: + variables = result.json() + except ValueError as ex: # would be a JSONDecodeError on Python 3, which subclasses ValueError + six.raise_from( + ValueError("The response body returned from starting playback did not contain valid JSON"), ex + ) + + return (recording_id, variables) -def stop_record_or_playback(test_id, recording_id): +def stop_record_or_playback(test_id, recording_id, test_output): + # type: (str, str, dict) -> None if is_live(): requests.post( RECORDING_STOP_URL, - headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, + headers={ + "x-recording-file": test_id, + "x-recording-id": recording_id, + "x-recording-save": "true", + "Content-Type": "application/json" + }, + json=test_output ) else: requests.post( @@ -104,10 +132,16 @@ def transform_request(request, recording_id): request.url = updated_target -def RecordedByProxy(func): +def recorded_by_proxy(test_func): + """Decorator that redirects network requests to target the azure-sdk-tools test proxy. Use with recorded tests. + + For more details and usage examples, refer to + https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/test_proxy_migration_guide.md + """ + def record_wrap(*args, **kwargs): test_id = get_test_id() - recording_id = start_record_or_playback(test_id) + recording_id, variables = start_record_or_playback(test_id) def transform_args(*args, **kwargs): copied_positional_args = list(args) @@ -118,7 +152,7 @@ def transform_args(*args, **kwargs): return tuple(copied_positional_args), kwargs trimmed_kwargs = {k: v for k, v in kwargs.items()} - trim_kwargs_from_test_function(func, trimmed_kwargs) + trim_kwargs_from_test_function(test_func, trimmed_kwargs) original_transport_func = RequestsTransport.send @@ -128,13 +162,22 @@ def combined_call(*args, **kwargs): RequestsTransport.send = combined_call - # call the modified function. + # call the modified function + # we define test_output before invoking the test so the variable is defined in case of an exception + test_output = None try: - value = func(*args, **trimmed_kwargs) + test_output = test_func(*args, variables=variables, **trimmed_kwargs) + except TypeError: + logger = logging.getLogger() + logger.info( + "This test can't accept variables as input. The test method should accept `**kwargs` and/or a " + "`variables` parameter to make use of recorded test variables." + ) + test_output = test_func(*args, **trimmed_kwargs) finally: RequestsTransport.send = original_transport_func - stop_record_or_playback(test_id, recording_id) + stop_record_or_playback(test_id, recording_id, test_output) - return value + return test_output return record_wrap