diff --git a/.gitignore b/.gitignore index 87b0050a20cc..0e86f299ec3a 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,6 @@ sdk/cosmos/azure-cosmos/test/test_config.py # env vars .env + +# local SSL certificate folder +.certificate \ No newline at end of file diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 4e0b0847cdbf..5703b16d04d3 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -44,3 +44,4 @@ $GetDocsMsMetadataForPackageFn = "Get-${Language}-DocsMsMetadataForPackage" $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageSpecificPackageInfo" $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview" +$TestProxyTrustCertFn = "Import-Dev-Cert-${Language}" \ No newline at end of file diff --git a/eng/common/scripts/trust-proxy-certificate.ps1 b/eng/common/scripts/trust-proxy-certificate.ps1 new file mode 100644 index 000000000000..144d304cfd18 --- /dev/null +++ b/eng/common/scripts/trust-proxy-certificate.ps1 @@ -0,0 +1,6 @@ +. $PSScriptRoot/common.ps1 + +if ($TestProxyTrustCertFn -and (Test-Path "Function:$TestProxyTrustCertFn")) +{ + &$TestProxyTrustCertFn +} \ No newline at end of file diff --git a/eng/common/testproxy/docker-start-proxy.ps1 b/eng/common/testproxy/docker-start-proxy.ps1 new file mode 100644 index 000000000000..4e5da37278a2 --- /dev/null +++ b/eng/common/testproxy/docker-start-proxy.ps1 @@ -0,0 +1,83 @@ + #!/usr/bin/env pwsh -c + +<# +.DESCRIPTION +Start the docker proxy container. If it is already running, quietly exit. Any other error should fail. +.PARAMETER Mode +"start" or "stop" to start up or stop the test-proxy instance. +.PARAMETER TargetFolder +The folder in which context the test proxy will be started. Defaults to current working directory. +#> +[CmdletBinding(SupportsShouldProcess = $true)] +param( + [ValidateSet("start", "stop")] + [String] + $Mode, + [String] + $TargetFolder = "." +) + +try { + docker --version | Out-Null +} +catch { + Write-Error "A invocation of docker --version failed. This indicates that docker is not properly installed or running." + Write-Error "Please check your docker invocation and try running the script again." +} + +$SELECTED_IMAGE_TAG = "1037115" +$CONTAINER_NAME = "ambitious_azsdk_test_proxy" +$LINUX_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-lin:${SELECTED_IMAGE_TAG}" +$WINDOWS_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-win:${SELECTED_IMAGE_TAG}" +$root = (Resolve-Path $TargetFolder).Path.Replace("`\", "/") + +function Get-Proxy-Container(){ + return (docker container ls -a --format "{{ json . }}" --filter "name=$CONTAINER_NAME" ` + | ConvertFrom-Json ` + | Select-Object -First 1) +} + + +$SelectedImage = $LINUX_IMAGE_SOURCE +$Initial = "" + +# most of the time, running this script on a windows machine will work just fine, as docker defaults to linux containers +# however, in CI, windows images default to _windows_ containers. We cannot swap them. We can tell if we're in a CI build by +# checking for the environment variable TF_BUILD. +if ($IsWindows -and $env:TF_BUILD){ + $SelectedImage = $WINDOWS_IMAGE_SOURCE + $Initial = "C:" +} + +if ($Mode -eq "start"){ + $proxyContainer = Get-Proxy-Container + + # if we already have one, we just need to check the state + if($proxyContainer){ + if ($proxyContainer.State -eq "running") + { + Write-Host "Discovered an already running instance of the test-proxy!. Exiting" + exit(0) + } + } + # else we need to create it + else { + Write-Host "Attempting creation of Docker host $CONTAINER_NAME" + Write-Host "docker container create -v `"${root}:${Initial}/etc/testproxy`" -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage" + docker container create -v "${root}:${Initial}/etc/testproxy" -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage + } + + Write-Host "Attempting start of Docker host $CONTAINER_NAME" + docker container start $CONTAINER_NAME +} + +if ($Mode -eq "stop"){ + $proxyContainer = Get-Proxy-Container + + if($proxyContainer){ + if($proxyContainer.State -eq "running"){ + Write-Host "Found a running instance of $CONTAINER_NAME, shutting it down." + docker container stop $CONTAINER_NAME + } + } +} \ No newline at end of file diff --git a/eng/common/testproxy/dotnet-devcert.crt b/eng/common/testproxy/dotnet-devcert.crt new file mode 100644 index 000000000000..e8575ea44564 --- /dev/null +++ b/eng/common/testproxy/dotnet-devcert.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSDCCAjCgAwIBAgIUPMKpJ/j10eQrcQBNnkImIaOYHakwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIxMDgwNTAwMzU1NloXDTIyMDgw +NTAwMzU1NlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAxe/ZseXgOTVoF7uTjX5Leknk95jIoyGc+VlxA8BhzGOr +r4u6VNQZRCMq+svHY36tW4+u/xHNe2kvbwy2mnS8cFFLfst+94qBZVJDBxSGZ9I/ +wekErNsjFsik4UrMvcC+ZlGPh7hb3f7tSx29tn1DIkAUXVnbZ6TT5s+mYRQpZ6fW +6kR3RNfc0A1IUM7Zs9yfNEr0O2H41P2HcLKoOPtvd7GvTQm9Ofh3srKvII+sZn/J +WH7r76oRQMX904mOMdryQwZLObsqX4dXIEbafKVSecB3PBVIhv8gVtJhcZbQP1pI +mMiWd6PHv46ZhGf7+cKnYUSa8Ia2t/wetK1wd00dFwIDAQABo4GRMIGOMA8GA1Ud +EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGmMBYGA1UdJQEB/wQMMAoGCCsGAQUF +BwMBMBcGA1UdEQEB/wQNMAuCCWxvY2FsaG9zdDA6BgorBgEEAYI3VAEBBCwMKkFT +UC5ORVQgQ29yZSBIVFRQUyBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTANBgkqhkiG +9w0BAQsFAAOCAQEAIj2VlBVcXGSly6KCBg6lgwFi+henWfSox77iuGAaAxDjN3jd +9lZahW4MPNLHKSrPRb4YNSLZ2jh7zdcttQrqd4qH65o1q56q5JrCmli99iIzY9Y8 +RdYyxK4Zzr31wjpsyFiWQfqJTuSFUUg9uDDj0negwEZLIGlt7nr12wflt2+QOJtD +byMeSZLbB5dPzn341DK0qfJEJMMgL0XsPEVZ3TQ6Alc9zq5wI608C/mXnz3xJE05 +UTYD8pRJJ/DyG0empvOVE8Sg93msHPquAbgqO9aqCpykgg/a8CFvI4wRdfvGEFlv +8XJKL8Y/PFsmFeO3axq3zUYKFVdc9Un4dFIaag== +-----END CERTIFICATE----- diff --git a/eng/common/testproxy/dotnet-devcert.pfx b/eng/common/testproxy/dotnet-devcert.pfx new file mode 100644 index 000000000000..28058ae4ce30 Binary files /dev/null and b/eng/common/testproxy/dotnet-devcert.pfx differ diff --git a/eng/common/testproxy/invoke-proxy.sh b/eng/common/testproxy/invoke-proxy.sh new file mode 100644 index 000000000000..be71a0e69212 --- /dev/null +++ b/eng/common/testproxy/invoke-proxy.sh @@ -0,0 +1 @@ +${TEST_PROXY_LOC}/test-proxy \ No newline at end of file diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml new file mode 100644 index 000000000000..97617b6fd08a --- /dev/null +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -0,0 +1,15 @@ +parameters: + rootFolder: '$(Build.SourcesDirectory)' + +steps: + - pwsh: | + $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1 + displayName: 'Language Specific Certificate Trust' + + - pwsh: | + $(Build.SourcesDirectory)/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" + displayName: 'Run the docker container' + + - pwsh: | + docker container ls -a + displayName: Check running container \ No newline at end of file diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml new file mode 100644 index 000000000000..9f24b0f0d527 --- /dev/null +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -0,0 +1,47 @@ +parameters: + rootFolder: '$(Build.SourcesDirectory)' + +steps: + - pwsh: | + $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1 + displayName: 'Language Specific Certificate Trust' + + - pwsh: | + Write-Host "##vso[task.setvariable variable=OriginalPath]$env:PATH" + displayName: 'Store Path Value' + + - pwsh: | + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]$(Build.SourcesDirectory)/eng/common/testproxy/dotnet-devcert.pfx" + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password" + displayName: 'Configure Kestrel Environment Variables' + + - task: UseDotNet@2 + displayName: "Use .NET Core SDK" + inputs: + packageType: sdk + version: 5.0.205 + + - pwsh: | + dotnet tool install azure.sdk.tools.testproxy ` + --tool-path $(Build.BinariesDirectory)/test-proxy ` + --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` + --version 1.0.0-dev.20210811.2 + displayName: "Install test-proxy" + + - pwsh: | + Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` + -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` + -NoNewWindow -PassThru + displayName: 'Run the testproxy - windows' + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) + + # nohup does NOT continue beyond the current session if you use it within powershell + - bash: | + sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & + displayName: "Run the testproxy - linux/mac" + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + workingDirectory: "${{ parameters.rootFolder }}" + + - pwsh: | + Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" + displayName: 'Restore .NET version by resetting path' \ No newline at end of file diff --git a/eng/pipelines/templates/steps/check-availability.yml b/eng/pipelines/templates/steps/check-availability.yml new file mode 100644 index 000000000000..7e9fb753220e --- /dev/null +++ b/eng/pipelines/templates/steps/check-availability.yml @@ -0,0 +1,4 @@ +steps: + - pwsh: | + Invoke-WebRequest -Uri "http://localhost:5000/Info/Available" + displayName: 'http://localhost:5000/Info/Available' \ No newline at end of file diff --git a/eng/pipelines/test-with-proxy.yml b/eng/pipelines/test-with-proxy.yml new file mode 100644 index 000000000000..d93941ea3629 --- /dev/null +++ b/eng/pipelines/test-with-proxy.yml @@ -0,0 +1,53 @@ +trigger: none + +variables: + PythonVersion: '3.9' + +jobs: + - job: 'win2019docker' + displayName: 'Windows Docker Run' + pool: + vmImage: 'windows-2019' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy/test-proxy-docker.yml + - template : /eng/pipelines/templates/steps/check-availability.yml + + - job: 'win2019tool' + displayName: 'Windows Tool Run' + pool: + vmImage: 'windows-2019' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy/test-proxy-tool.yml + - template : /eng/pipelines/templates/steps/check-availability.yml + + - job: 'linux2020docker' + displayName: 'Linux Docker Run' + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy/test-proxy-docker.yml + - template : /eng/pipelines/templates/steps/check-availability.yml + + - job: 'linux2020tool' + displayName: 'Linux Tool Run' + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/testproxy/test-proxy-tool.yml + - template : /eng/pipelines/templates/steps/check-availability.yml \ No newline at end of file diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 31e70001673b..2dcab179bdd9 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -282,3 +282,7 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null) return $null } } +function Import-Dev-Cert-python +{ + Write-Host "Certificate action here!" +} diff --git a/scripts/install_certificate.py b/scripts/install_certificate.py new file mode 100644 index 000000000000..d721d607513d --- /dev/null +++ b/scripts/install_certificate.py @@ -0,0 +1,33 @@ +import requests +import os + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..")) +CERT_URL = "https://raw.githubusercontent.com/Azure/azure-sdk-tools/main/tools/test-proxy/docker/dev_certificate/dotnet-devcert.crt" +EXISTING_ROOT_PEM = requests.certs.where() +LOCAL_FILENAME = CERT_URL.split('/')[-1].split(".")[0] + ".pem" +LOCAL_LOCATION = os.path.join(root_dir, '.certificate', LOCAL_FILENAME) + +def download_cert_file(): + content = requests.get(CERT_URL) + + if content.status_code == 200: + with open(LOCAL_LOCATION, 'w') as f: + f.write(content.text) + +def combine_cert_file(): + with open(EXISTING_ROOT_PEM, 'r') as f: + content = f.readlines(); + + with open(LOCAL_LOCATION, 'a') as f: + f.writelines(content) + +if __name__ == "__main__": + download_cert_file() + combine_cert_file() + + print("Set the following certificate paths:") + print("\tSSL_CERT_DIR={}".format(os.path.dirname(LOCAL_LOCATION))) + print("\tREQUESTS_CA_BUNDLE={}".format(LOCAL_LOCATION)) + + print("\nBe aware that REQUESTS_CA_BUNDLE should only be modified ") + diff --git a/sdk/storage/azure-storage-blob/tests/_shared/testcase.py b/sdk/storage/azure-storage-blob/tests/_shared/testcase.py index 8c5db62d4efa..bdefdef82054 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/testcase.py +++ b/sdk/storage/azure-storage-blob/tests/_shared/testcase.py @@ -9,6 +9,7 @@ import time import os import logging +import pdb from devtools_testutils import ( AzureMgmtTestCase, AzureMgmtPreparer, @@ -32,11 +33,10 @@ import pytest +ENABLE_LOGGING = True from devtools_testutils.storage import StorageTestCase LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s' -os.environ['AZURE_STORAGE_ACCOUNT_NAME'] = STORAGE_ACCOUNT_NAME -os.environ['AZURE_STORAGE_ACCOUNT_KEY'] = STORAGE_ACCOUNT_KEY os.environ['AZURE_TEST_RUN_LIVE'] = os.environ.get('AZURE_TEST_RUN_LIVE', None) or RUN_IN_LIVE os.environ['AZURE_SKIP_LIVE_RECORDING'] = os.environ.get('AZURE_SKIP_LIVE_RECORDING', None) or SKIP_LIVE_RECORDING @@ -162,9 +162,11 @@ def storage_account(): try: if i_need_to_create_rg: + logging.info("I NEED TO CREATE RG") rg_name, rg_kwargs = rg_preparer._prepare_create_resource(test_case) rg = rg_kwargs['resource_group'] else: + logging.info("I DONT NEED TO CREATE RG") rg_name = existing_rg_name or "no_rg_needed" rg = FakeResource( name=rg_name, @@ -172,9 +174,10 @@ def storage_account(): ) StorageTestCase._RESOURCE_GROUP = rg + try: if got_storage_info_from_env: - + logging.info("EXISTING INFO IN ENV") if storage_connection_string: storage_connection_string_parts = dict([ part.split('=', 1) @@ -183,6 +186,7 @@ def storage_account(): storage_account = None if existing_storage_name: + logging.info("EXISTING NAME IS {}".format(existing_storage_name)) storage_name = existing_storage_name storage_account = StorageAccount( location=location, diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_client.py b/sdk/storage/azure-storage-blob/tests/test_blob_client.py index 5956f7a92e85..248cac117eb1 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_client.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_client.py @@ -15,7 +15,7 @@ ContainerClient, BlobClient, ) -from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer +from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer, RecordedByProxy from _shared.testcase import GlobalStorageAccountPreparer from devtools_testutils.storage import StorageTestCase @@ -48,9 +48,9 @@ def validate_standard_account_endpoints(self, service, url_type, name, storage_a # --Direct Parameters Test Cases -------------------------------------------- @GlobalStorageAccountPreparer() + @RecordedByProxy def test_create_service_with_key(self, resource_group, location, storage_account, storage_account_key): # Arrange - for client, url in SERVICES.items(): # Act service = client( diff --git a/sdk/tables/azure-data-tables/tests/async_preparers.py b/sdk/tables/azure-data-tables/tests/async_preparers.py index f8768e3b8c6b..17645fb1092e 100644 --- a/sdk/tables/azure-data-tables/tests/async_preparers.py +++ b/sdk/tables/azure-data-tables/tests/async_preparers.py @@ -1,6 +1,7 @@ import time from azure.core.credentials import AzureNamedKeyCredential +from devtools_testutils.aio import RecordedByProxyAsync from azure.core.exceptions import HttpResponseError from devtools_testutils import is_live @@ -10,6 +11,7 @@ def cosmos_decorator_async(func, **kwargs): @CosmosPreparer() + @RecordedByProxyAsync async def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_cosmos_account_key") name = kwargs.pop("tables_cosmos_account_name") @@ -28,6 +30,7 @@ async def wrapper(*args, **kwargs): def tables_decorator_async(func, **kwargs): @TablesPreparer() + @RecordedByProxyAsync async def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_storage_account_key") name = kwargs.pop("tables_storage_account_name") diff --git a/sdk/tables/azure-data-tables/tests/preparers.py b/sdk/tables/azure-data-tables/tests/preparers.py index 0ab2b63521b9..3cb81ecbc336 100644 --- a/sdk/tables/azure-data-tables/tests/preparers.py +++ b/sdk/tables/azure-data-tables/tests/preparers.py @@ -4,7 +4,7 @@ from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import HttpResponseError -from devtools_testutils import PowerShellPreparer, is_live +from devtools_testutils import PowerShellPreparer, is_live, RecordedByProxy CosmosPreparer = functools.partial( PowerShellPreparer, @@ -16,7 +16,7 @@ TablesPreparer = functools.partial( PowerShellPreparer, "tables", - tables_storage_account_name="fake_table_account", + tables_storage_account_name="faketableaccount", tables_primary_storage_account_key="faketablesaccountkey", ) @@ -37,6 +37,7 @@ def trim_kwargs_from_test_function(fn, kwargs): def tables_decorator(func, **kwargs): @TablesPreparer() + @RecordedByProxy def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_storage_account_key") name = kwargs.pop("tables_storage_account_name") @@ -55,6 +56,7 @@ def wrapper(*args, **kwargs): def cosmos_decorator(func, **kwargs): @CosmosPreparer() + @RecordedByProxy def wrapper(*args, **kwargs): key = kwargs.pop("tables_primary_cosmos_account_key") name = kwargs.pop("tables_cosmos_account_name") diff --git a/sdk/tables/azure-data-tables/tests/recordings/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f b/sdk/tables/azure-data-tables/tests/recordings/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f new file mode 100644 index 000000000000..b42c6f18aa45 --- /dev/null +++ b/sdk/tables/azure-data-tables/tests/recordings/test_table_service_stats.py.TestTableServiceStats.test_table_service_stats_f @@ -0,0 +1,35 @@ +{ + "Entries": [ + { + "RequestUri": "https://faketableaccount-secondary.table.core.windows.net/?restype=service\u0026comp=stats", + "RequestMethod": "GET", + "RequestHeaders": { + "Accept": "application/xml", + "Accept-Encoding": "gzip, deflate", + "Authorization": "Sanitized", + "Connection": "keep-alive", + "Date": "Thu, 12 Aug 2021 17:45:09 GMT", + "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.0 (Windows-10-10.0.19041-SP0)", + "x-ms-client-request-id": "09dbc5e2-fb95-11eb-a228-c8348e51ab51", + "x-ms-date": "Thu, 12 Aug 2021 17:45:09 GMT", + "x-ms-version": "2019-02-02" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Content-Type": "application/xml", + "Date": "Thu, 12 Aug 2021 17:45:08 GMT", + "Server": [ + "Windows-Azure-Table/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "Transfer-Encoding": "chunked", + "x-ms-client-request-id": "09dbc5e2-fb95-11eb-a228-c8348e51ab51", + "x-ms-request-id": "a7683a07-2002-002f-0ba1-8fbf82000000", + "x-ms-version": "2019-02-02" + }, + "ResponseBody": "\uFEFF\u003C?xml version=\u00221.0\u0022 encoding=\u0022utf-8\u0022?\u003E\u003CStorageServiceStats\u003E\u003CGeoReplication\u003E\u003CStatus\u003Elive\u003C/Status\u003E\u003CLastSyncTime\u003EThu, 12 Aug 2021 17:42:04 GMT\u003C/LastSyncTime\u003E\u003C/GeoReplication\u003E\u003C/StorageServiceStats\u003E" + } + ], + "Variables": {} +} \ No newline at end of file diff --git a/sdk/tables/azure-data-tables/tests/test_retry.py b/sdk/tables/azure-data-tables/tests/test_retry.py index bc7faf72d056..ebb64074880e 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry.py +++ b/sdk/tables/azure-data-tables/tests/test_retry.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase, ResponseCallback +from devtools_testutils import AzureRecordedTestCase, ResponseCallback from azure.core.exceptions import ( HttpResponseError, @@ -33,7 +33,7 @@ def send(self, request, **kwargs): return response # --Test Class ----------------------------------------------------------------- -class StorageRetryTest(AzureTestCase, TableTestCase): +class TestStorageRetry(AzureRecordedTestCase, TableTestCase): def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key, url='table', default_table=True, **kwargs): self.table_name = self.get_resource_name('uttable') @@ -92,7 +92,7 @@ def test_retry_on_timeout(self, tables_storage_account_name, tables_primary_stor default_table=False, retry_mode=RetryMode.Exponential, retry_backoff_factor=1 - ) + ) callback = ResponseCallback(status=200, new_status=408).override_first_status try: diff --git a/sdk/tables/azure-data-tables/tests/test_retry_async.py b/sdk/tables/azure-data-tables/tests/test_retry_async.py index a389686e2617..77f77332ed1c 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry_async.py +++ b/sdk/tables/azure-data-tables/tests/test_retry_async.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase, ResponseCallback +from devtools_testutils import AzureRecordedTestCase, AzureTestCase, ResponseCallback from azure.core.exceptions import ( @@ -38,7 +38,7 @@ async def send(self, request, **kwargs): # --Test Class ----------------------------------------------------------------- -class StorageRetryTest(AzureTestCase, AsyncTableTestCase): +class TestStorageRetry(AzureRecordedTestCase, AsyncTableTestCase): async def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key, url='table', default_table=True, **kwargs): self.table_name = self.get_resource_name('uttable') @@ -79,7 +79,8 @@ async def test_retry_on_timeout_async(self, tables_storage_account_name, tables_ tables_primary_storage_account_key, default_table=False, retry_mode=RetryMode.Exponential, - retry_backoff_factor=1) + retry_backoff_factor=1 + ) callback = ResponseCallback(status=200, new_status=408).override_first_status diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index 3b2a3c2787a0..1487867f49bd 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -8,7 +8,7 @@ import pytest from datetime import datetime, timedelta -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import ( ResourceTypes, @@ -26,12 +26,12 @@ from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential from azure.core.exceptions import ResourceExistsError -from _shared.testcase import TableTestCase, TEST_TABLE_PREFIX +from _shared.testcase import TableTestCase from preparers import tables_decorator, tables_decorator # ------------------------------------------------------------------------------ -class StorageTableTest(AzureTestCase, TableTestCase): +class TestTable(AzureRecordedTestCase, TableTestCase): @tables_decorator def test_create_properties(self, tables_storage_account_name, tables_primary_storage_account_key): # # Arrange @@ -428,7 +428,6 @@ def test_account_sas(self, tables_storage_account_name, tables_primary_storage_a service = TableServiceClient(credential=token, endpoint=account_url) # Act - sas_table = service.get_table_client(table.table_name) entities = list(sas_table.list_entities()) diff --git a/sdk/tables/azure-data-tables/tests/test_table_async.py b/sdk/tables/azure-data-tables/tests/test_table_async.py index c92d41abe346..f5a45cb01dd5 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_async.py @@ -2,7 +2,7 @@ import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential from azure.core.exceptions import ResourceExistsError @@ -20,7 +20,7 @@ from async_preparers import tables_decorator_async -class TableTestAsync(AzureTestCase, AsyncTableTestCase): +class TableTestAsync(AzureRecordedTestCase, AsyncTableTestCase): @tables_decorator_async async def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange @@ -383,7 +383,7 @@ async def test_account_sas(self, tables_storage_account_name, tables_primary_sto await tsc.delete_table(table.table_name) -class TestTablesUnitTest(AsyncTableTestCase): +class TestTablesUnit(AsyncTableTestCase): tables_storage_account_name = "fake_storage_account" tables_primary_storage_account_key = "fakeXMZjnGsZGvd4bVr3Il5SeHA" credential = AzureNamedKeyCredential(name=tables_storage_account_name, key=tables_primary_storage_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch.py b/sdk/tables/azure-data-tables/tests/test_table_batch.py index fca3e2ae0489..54f8c36db1a1 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch.py @@ -12,7 +12,7 @@ import os import sys -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential @@ -43,7 +43,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableBatchTest(AzureTestCase, TableTestCase): +class TestTableBatch(AzureRecordedTestCase, TableTestCase): @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3") @tables_decorator def test_batch_single_insert(self, tables_storage_account_name, tables_primary_storage_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_async.py b/sdk/tables/azure-data-tables/tests/test_table_batch_async.py index b965d26d487c..286e9c256752 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_async.py @@ -12,7 +12,7 @@ import os import sys -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureSasCredential, AzureNamedKeyCredential @@ -38,7 +38,7 @@ from async_preparers import tables_decorator_async -class StorageTableBatchTest(AzureTestCase, AsyncTableTestCase): +class TestTableBatch(AzureRecordedTestCase, AsyncTableTestCase): @tables_decorator_async async def test_batch_single_insert(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py index 76abebca9cdb..95150fc65a46 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos.py @@ -11,7 +11,7 @@ import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureSasCredential @@ -36,7 +36,7 @@ from preparers import cosmos_decorator -class StorageTableClientTest(AzureTestCase, TableTestCase): +class TestTableClient(AzureRecordedTestCase, TableTestCase): @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3") @cosmos_decorator def test_batch_insert(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py index 6d7c4165c983..b1ea570a54b3 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_batch_cosmos_async.py @@ -10,7 +10,7 @@ import sys import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureSasCredential, AzureNamedKeyCredential @@ -40,7 +40,7 @@ TEST_TABLE_PREFIX = 'table' #------------------------------------------------------------------------------ -class StorageTableBatchTest(AzureTestCase, AsyncTableTestCase): +class TestTableBatch(AzureRecordedTestCase, AsyncTableTestCase): @cosmos_decorator_async async def test_batch_single_insert(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): # Arrange diff --git a/sdk/tables/azure-data-tables/tests/test_table_client.py b/sdk/tables/azure-data-tables/tests/test_table_client.py index 29b831fe3669..21b1032c7cfa 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client.py @@ -6,7 +6,7 @@ import pytest import platform -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableServiceClient, TableClient from azure.data.tables import __version__ as VERSION @@ -29,7 +29,7 @@ _CONNECTION_ENDPOINTS_SECONDARY = {'table': 'TableSecondaryEndpoint', 'cosmos': 'TableSecondaryEndpoint'} -class TestTableClient(AzureTestCase, TableTestCase): +class TestTableClient(AzureRecordedTestCase, TableTestCase): @tables_decorator def test_user_agent_custom(self, tables_storage_account_name, tables_primary_storage_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_client_async.py b/sdk/tables/azure-data-tables/tests/test_table_client_async.py index 01eef4c2ccf0..8f004268c16c 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client_async.py @@ -7,7 +7,7 @@ import pytest import platform -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential from azure.data.tables.aio import TableServiceClient, TableClient @@ -15,7 +15,6 @@ from _shared.asynctestcase import AsyncTableTestCase from async_preparers import tables_decorator_async -from devtools_testutils import AzureTestCase # ------------------------------------------------------------------------------ SERVICES = { TableServiceClient: 'table', @@ -27,7 +26,7 @@ _CONNECTION_ENDPOINTS_SECONDARY = {'table': 'TableSecondaryEndpoint'} -class TestTableClient(AzureTestCase, AsyncTableTestCase): +class TestTableClient(AzureRecordedTestCase, AsyncTableTestCase): @tables_decorator_async async def test_user_agent_default_async(self, tables_storage_account_name, tables_primary_storage_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos.py index c24f9dfe9c2e..3f622d61c8bc 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos.py @@ -7,7 +7,7 @@ import platform import sys -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableServiceClient, TableClient from azure.data.tables import __version__ as VERSION @@ -29,7 +29,7 @@ _CONNECTION_ENDPOINTS_SECONDARY = {'table': 'TableSecondaryEndpoint', 'cosmos': 'TableSecondaryEndpoint'} -class TestTableClient(AzureTestCase, TableTestCase): +class TestTableClient(AzureRecordedTestCase, TableTestCase): @pytest.mark.skipif(sys.version_info < (3, 0), reason="Malformed string") @cosmos_decorator diff --git a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py index e380d395f8fe..a1670aea83b8 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_client_cosmos_async.py @@ -7,7 +7,7 @@ import pytest import platform -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables.aio import TableServiceClient, TableClient from azure.data.tables import __version__ as VERSION @@ -15,7 +15,6 @@ from _shared.asynctestcase import AsyncTableTestCase from _shared.testcase import SLEEP_DELAY from async_preparers import cosmos_decorator_async -from devtools_testutils import AzureTestCase # ------------------------------------------------------------------------------ @@ -28,7 +27,7 @@ _CONNECTION_ENDPOINTS_SECONDARY = {'table': 'TableSecondaryEndpoint', 'cosmos': 'TableSecondaryEndpoint'} -class TestTableClient(AzureTestCase, AsyncTableTestCase): +class TestTableClient(AzureRecordedTestCase, AsyncTableTestCase): @cosmos_decorator_async async def test_user_agent_default_async(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_cosmos.py index d1c63c9acce2..851b9d7fa52b 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_cosmos.py @@ -6,7 +6,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ResourceExistsError @@ -19,7 +19,7 @@ TEST_TABLE_PREFIX = 'pytablesync' # ------------------------------------------------------------------------------ -class StorageTableTest(AzureTestCase, TableTestCase): +class TestStorageTable(AzureRecordedTestCase, TableTestCase): @cosmos_decorator def test_create_table(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): @@ -185,7 +185,7 @@ def test_delete_table_with_non_existing_table_fail_not_exist(self, tables_cosmos ts.delete_table(table_name) -class TestTableUnitTest(TableTestCase): +class TestTableUnit(TableTestCase): tables_cosmos_account_name = "fake_storage_account" tables_primary_cosmos_account_key = "fakeXMZjnGsZGvd4bVr3Il5SeHA" credential = AzureNamedKeyCredential(name=tables_cosmos_account_name, key=tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py index de3006c07ea9..82ae8cd7a062 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_cosmos_async.py @@ -1,6 +1,6 @@ import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ResourceExistsError @@ -14,7 +14,7 @@ # ------------------------------------------------------------------------------ -class TableTestAsync(AzureTestCase, AsyncTableTestCase): +class TestTableAsync(AzureRecordedTestCase, AsyncTableTestCase): @cosmos_decorator_async async def test_create_table(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): # Arrange diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity.py b/sdk/tables/azure-data-tables/tests/test_table_entity.py index 06c67937b5f2..d2b1d6cb28df 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity.py @@ -12,7 +12,7 @@ from enum import Enum from math import isnan -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import ( TableServiceClient, @@ -39,7 +39,7 @@ # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureTestCase, TableTestCase): +class TestTableEntity(AzureRecordedTestCase, TableTestCase): @tables_decorator def test_url_encoding_at_symbol(self, tables_storage_account_name, tables_primary_storage_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_async.py b/sdk/tables/azure-data-tables/tests/test_table_entity_async.py index 54b32c57ec47..675a9bc26050 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_async.py @@ -11,7 +11,7 @@ from dateutil.tz import tzutc, tzoffset from math import isnan -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureSasCredential @@ -36,7 +36,7 @@ from _shared.asynctestcase import AsyncTableTestCase from async_preparers import tables_decorator_async -class StorageTableEntityTest(AzureTestCase, AsyncTableTestCase): +class TestTableEntity(AzureRecordedTestCase, AsyncTableTestCase): @tables_decorator_async async def test_url_encoding_at_symbol(self, tables_storage_account_name, tables_primary_storage_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py index 9d6d1e319962..076d4d8056c3 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos.py @@ -11,7 +11,7 @@ from dateutil.tz import tzutc, tzoffset from math import isnan -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core import MatchConditions from azure.core.credentials import AzureSasCredential @@ -37,7 +37,7 @@ # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureTestCase, TableTestCase): +class TestTableEntity(AzureRecordedTestCase, TableTestCase): @cosmos_decorator def test_url_encoding_at_symbol(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py index 0ed16fecfdca..024dcd0baebd 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_entity_cosmos_async.py @@ -10,7 +10,7 @@ from dateutil.tz import tzutc, tzoffset from math import isnan -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import ( TableEntity, @@ -37,7 +37,7 @@ # TODO: change to `with table_client as client:` to close sessions # ------------------------------------------------------------------------------ -class StorageTableEntityTest(AzureTestCase, AsyncTableTestCase): +class TestTableEntity(AzureRecordedTestCase, AsyncTableTestCase): @cosmos_decorator_async async def test_url_encoding_at_symbol(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties.py index b89895ca1abb..c57ad7a208bd 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties.py @@ -8,7 +8,7 @@ import time import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import ( TableServiceClient, @@ -25,7 +25,7 @@ # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureTestCase, TableTestCase): +class TestTableServiceProperties(AzureRecordedTestCase, TableTestCase): @tables_decorator def test_table_service_properties(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py index f1b7357b5fe7..884cb39068a6 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_async.py @@ -8,7 +8,7 @@ import time import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.exceptions import HttpResponseError @@ -21,7 +21,7 @@ # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureTestCase, TableTestCase): +class TestTableServiceProperties(AzureRecordedTestCase, TableTestCase): @tables_decorator_async async def test_table_service_properties_async(self, tables_storage_account_name, tables_primary_storage_account_key): # Arrange diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py index 5a2b17b43701..7074e6cbbdc1 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos.py @@ -7,7 +7,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.core.exceptions import HttpResponseError @@ -18,11 +18,11 @@ TableCorsRule ) -from _shared.testcase import TableTestCase, SLEEP_DELAY +from _shared.testcase import TableTestCase from preparers import cosmos_decorator # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureTestCase, TableTestCase): +class TestTableServiceProperties(AzureRecordedTestCase, TableTestCase): @cosmos_decorator def test_too_many_cors_rules(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): tsc = TableServiceClient(self.account_url(tables_cosmos_account_name, "cosmos"), credential=tables_primary_cosmos_account_key) diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py index d9ee136c2a61..8b96f999eac1 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_properties_cosmos_async.py @@ -7,7 +7,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableAnalyticsLogging, TableMetrics, TableRetentionPolicy, TableCorsRule from azure.data.tables.aio import TableServiceClient @@ -18,7 +18,7 @@ from async_preparers import cosmos_decorator_async # ------------------------------------------------------------------------------ -class TableServicePropertiesTest(AzureTestCase, AsyncTableTestCase): +class TestTableServiceProperties(AzureRecordedTestCase, AsyncTableTestCase): @cosmos_decorator_async async def test_too_many_cors_rules_async(self, tables_cosmos_account_name, tables_primary_cosmos_account_key): # Arrange diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py index 6658e702d677..f76c5144c2aa 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats.py @@ -3,14 +3,17 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase, ProxyRecordingSanitizer from azure.data.tables import TableServiceClient from _shared.testcase import TableTestCase from preparers import tables_decorator # --Test Class ----------------------------------------------------------------- -class TableServiceStatsTest(AzureTestCase, TableTestCase): +class TestTableServiceStats(AzureRecordedTestCase, TableTestCase): + + def setup_method(self): + self.add_sanitizer(ProxyRecordingSanitizer.URI, value="faketableaccount") # --Test cases per service --------------------------------------- @tables_decorator diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py index 2aeffef1e965..0bb3d3f10bac 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats_async.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables.aio import TableServiceClient @@ -19,7 +19,7 @@ '> ' -class TableServiceStatsTest(AzureTestCase, AsyncTableTestCase): +class TestTableServiceStats(AzureRecordedTestCase, AsyncTableTestCase): @staticmethod def override_response_body_with_unavailable_status(response): diff --git a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py index 8c6b41f3eac2..e8da2db3e6b2 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py +++ b/sdk/tables/azure-data-tables/tests/test_table_service_stats_cosmos.py @@ -5,10 +5,10 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureRecordedTestCase from azure.data.tables import TableServiceClient -from _shared.testcase import TableTestCase, SLEEP_DELAY +from _shared.testcase import TableTestCase from preparers import cosmos_decorator SERVICE_UNAVAILABLE_RESP_BODY = 'liveWed, 19 Jan 2021 22:28:43 GMT ' -class TableServiceStatsTest(AzureTestCase, AsyncTableTestCase): +class TestTableServiceStats(AzureRecordedTestCase, AsyncTableTestCase): @staticmethod def override_response_body_with_unavailable_status(response): diff --git a/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py b/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py index 1bc992962882..73f1150db866 100644 --- a/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py +++ b/tools/azure-devtools/src/azure_devtools/scenario_tests/preparers.py @@ -135,7 +135,9 @@ def _preparer_wrapper(test_class_instance, **kwargs): ) if test_class_instance.is_live: - test_class_instance.scrubber.register_name_pair(resource_name, self.moniker) + # Adding this for new proxy testcase + if hasattr(test_class_instance, "scrubber"): + test_class_instance.scrubber.register_name_pair(resource_name, self.moniker) # We shouldn't trim the same kwargs that we use for deletion, # we may remove some of the variables we needed to do the delete. diff --git a/tools/azure-sdk-tools/devtools_testutils/__init__.py b/tools/azure-sdk-tools/devtools_testutils/__init__.py index 640da9b62531..2c54da0935b5 100644 --- a/tools/azure-sdk-tools/devtools_testutils/__init__.py +++ b/tools/azure-sdk-tools/devtools_testutils/__init__.py @@ -1,5 +1,7 @@ from .mgmt_testcase import AzureMgmtTestCase, AzureMgmtPreparer +from .azure_recorded_testcase import AzureRecordedTestCase from .azure_testcase import AzureTestCase, is_live, get_region_override +from .enums import ProxyRecordingSanitizer from .resource_testcase import ( FakeResource, ResourceGroupPreparer, @@ -14,10 +16,12 @@ ) from .keyvault_preparer import KeyVaultPreparer from .powershell_preparer import PowerShellPreparer +from .proxy_testcase import RecordedByProxy from .helpers import ResponseCallback, RetryCounter from .fake_credential import FakeTokenCredential __all__ = [ + "AzureRecordedTestCase", "AzureMgmtTestCase", "AzureMgmtPreparer", "FakeResource", @@ -33,7 +37,9 @@ "RandomNameResourceGroupPreparer", "CachedResourceGroupPreparer", "PowerShellPreparer", + "ProxyRecordingSanitizer", + "RecordedByProxy", "ResponseCallback", "RetryCounter", - "FakeTokenCredential", + "FakeTokenCredential" ] diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py b/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py new file mode 100644 index 000000000000..5265d80fe58e --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/aio/__init__.py @@ -0,0 +1,3 @@ +from .proxy_testcase_async import RecordedByProxyAsync + +__all__ = ["RecordedByProxyAsync"] diff --git a/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py new file mode 100644 index 000000000000..11c8f9e44395 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/aio/proxy_testcase_async.py @@ -0,0 +1,50 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from azure.core.pipeline.transport import AioHttpTransport + +from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function +from ..proxy_testcase import ( + get_test_id, + start_record_or_playback, + transform_request, + stop_record_or_playback, +) + + +def RecordedByProxyAsync(func): + async def record_wrap(*args, **kwargs): + test_id = get_test_id() + recording_id = start_record_or_playback(test_id) + + def transform_args(*args, **kwargs): + copied_positional_args = list(args) + request = copied_positional_args[1] + + transform_request(request, recording_id) + + return tuple(copied_positional_args), kwargs + + trimmed_kwargs = {k: v for k, v in kwargs.items()} + trim_kwargs_from_test_function(func, trimmed_kwargs) + + original_func = AioHttpTransport.send + + async def combined_call(*args, **kwargs): + adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) + return await original_func(*adjusted_args, **adjusted_kwargs) + + AioHttpTransport.send = combined_call + + # call the modified function. + try: + value = await func(*args, **trimmed_kwargs) + finally: + AioHttpTransport.send = original_func + stop_record_or_playback(test_id, recording_id) + + return value + + return record_wrap diff --git a/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py new file mode 100644 index 000000000000..e045da62966a --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/azure_recorded_testcase.py @@ -0,0 +1,265 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import functools +import logging +import os +import os.path +import requests +import six +import sys +import time +from typing import TYPE_CHECKING + +from dotenv import load_dotenv, find_dotenv + +from azure_devtools.scenario_tests import AzureTestError +from azure_devtools.scenario_tests.config import TestConfig +from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function + +from . import mgmt_settings_fake as fake_settings +from .azure_testcase import _is_autorest_v3, get_resource_name, get_qualified_method_name +from .config import PROXY_URL +from .enums import ProxyRecordingSanitizer + +try: + # Try to import the AsyncFakeCredential, if we cannot assume it is Python 2 + from .fake_async_credential import AsyncFakeCredential +except SyntaxError: + pass + +if TYPE_CHECKING: + from typing import Optional + + +load_dotenv(find_dotenv()) + + +def is_live(): + """A module version of is_live, that could be used in pytest marker.""" + if not hasattr(is_live, "_cache"): + is_live._cache = TestConfig().record_mode + return is_live._cache + + +class AzureRecordedTestCase(object): + @property + def settings(self): + if self.is_live: + if self._real_settings: + return self._real_settings + else: + raise AzureTestError("Need a mgmt_settings_real.py file to run tests live.") + else: + return self._fake_settings + + def _load_settings(self): + try: + from . import mgmt_settings_real as real_settings + + return fake_settings, real_settings + except ImportError: + return fake_settings, None + + @property + def is_live(self): + return is_live() + + @property + def qualified_test_name(self): + return get_qualified_method_name(self, "method_name") + + @property + def in_recording(self): + return self.is_live + + # TODO: This needs to be removed, recording processors are handled on the proxy side, but + # this is needed for the preparers + @property + def recording_processors(self): + return [] + + def add_sanitizer(self, sanitizer, regex=None, value=None): + # type: (ProxyRecordingSanitizer, Optional[str], Optional[str]) -> None + if sanitizer == ProxyRecordingSanitizer.URI: + requests.post( + "{}/Admin/AddSanitizer".format(PROXY_URL), + headers={"x-abstraction-identifier": ProxyRecordingSanitizer.URI.value}, + json={ + "regex": regex or "[a-z]+(?=(?:-secondary)\\.(?:table|blob|queue)\\.core\\.windows\\.net)", + "value": value or "fakevalue" + }, + ) + + def is_playback(self): + return not self.is_live + + def get_settings_value(self, key): + key_value = os.environ.get("AZURE_" + key, None) + + if key_value and self._real_settings and getattr(self._real_settings, key) != key_value: + raise ValueError( + "You have both AZURE_{key} env variable and mgmt_settings_real.py for {key} to different values".format( + key=key + ) + ) + + if not key_value: + try: + key_value = getattr(self.settings, key) + except Exception as ex: + six.raise_from(ValueError("Could not get {}".format(key)), ex) + return key_value + + def get_credential(self, client_class, **kwargs): + tenant_id = os.environ.get("AZURE_TENANT_ID", getattr(self._real_settings, "TENANT_ID", None)) + client_id = os.environ.get("AZURE_CLIENT_ID", getattr(self._real_settings, "CLIENT_ID", None)) + secret = os.environ.get("AZURE_CLIENT_SECRET", getattr(self._real_settings, "CLIENT_SECRET", None)) + is_async = kwargs.pop("is_async", False) + + if tenant_id and client_id and secret and self.is_live: + if _is_autorest_v3(client_class): + # Create azure-identity class + from azure.identity import ClientSecretCredential + + if is_async: + from azure.identity.aio import ClientSecretCredential + return ClientSecretCredential(tenant_id=tenant_id, client_id=client_id, client_secret=secret) + else: + # Create msrestazure class + from msrestazure.azure_active_directory import ( + ServicePrincipalCredentials, + ) + + return ServicePrincipalCredentials(tenant=tenant_id, client_id=client_id, secret=secret) + else: + if _is_autorest_v3(client_class): + if is_async: + if self.is_live: + raise ValueError( + "Async live doesn't support mgmt_setting_real, please set AZURE_TENANT_ID," + "AZURE_CLIENT_ID, AZURE_CLIENT_SECRET" + ) + return AsyncFakeCredential() + else: + return self.settings.get_azure_core_credentials() + else: + return self.settings.get_credentials() + + def create_client_from_credential(self, client_class, credential, **kwargs): + + # Real client creation + # TODO decide what is the final argument for that + # if self.is_playback(): + # kwargs.setdefault("polling_interval", 0) + if _is_autorest_v3(client_class): + kwargs.setdefault("logging_enable", True) + client = client_class(credential=credential, **kwargs) + else: + client = client_class(credentials=credential, **kwargs) + + if self.is_playback(): + try: + client._config.polling_interval = 0 # FIXME in azure-mgmt-core, make this a kwargs + except AttributeError: + pass + + if hasattr(client, "config"): # Autorest v2 + if self.is_playback(): + client.config.long_running_operation_timeout = 0 + client.config.enable_http_logger = True + return client + + def create_basic_client(self, client_class, **kwargs): + """ DO NOT USE ME ANYMORE.""" + logger = logging.getLogger() + logger.warning( + "'create_basic_client' will be deprecated in the future. It is recommended that you use \ + 'get_credential' and 'create_client_from_credential' to create your client." + ) + + credentials = self.get_credential(client_class) + return self.create_client_from_credential(client_class, credentials, **kwargs) + + def create_random_name(self, name): + unique_test_name = os.getenv("PYTEST_CURRENT_TEST").encode("utf-8") + return get_resource_name(name, unique_test_name) + + def get_resource_name(self, name): + """Alias to create_random_name for back compatibility.""" + return self.create_random_name(name) + + def get_replayable_random_resource_name(self, name): + """In a replay scenario (not live), gives the static moniker. In the random scenario, gives generated name.""" + if self.is_live: + created_name = self.create_random_name(name) + self.scrubber.register_name_pair(created_name, name) + return name + + def get_preparer_resource_name(self, prefix): + """Random name generation for use by preparers. + + If prefix is a blank string, use the fully qualified test name instead. + This is what legacy tests do for resource groups.""" + return self.get_resource_name(prefix) + + @staticmethod + def await_prepared_test(test_fn): + """Synchronous wrapper for async test methods. Used to avoid making changes + upstream to AbstractPreparer, which only awaits async tests that use preparers. + (Add @AzureTestCase.await_prepared_test decorator to async tests without preparers) + + # Note: this will only be needed so long as we maintain unittest.TestCase in our + test-class inheritance chain. + """ + + if sys.version_info < (3, 5): + raise ImportError("Async wrapper is not needed for Python 2.7 code.") + + import asyncio + + @functools.wraps(test_fn) + def run(test_class_instance, *args, **kwargs): + trim_kwargs_from_test_function(test_fn, kwargs) + loop = asyncio.get_event_loop() + return loop.run_until_complete(test_fn(test_class_instance, **kwargs)) + + return run + + def sleep(self, seconds): + if self.is_live: + time.sleep(seconds) + + def generate_sas(self, *args, **kwargs): + sas_func = args[0] + sas_func_pos_args = args[1:] + + fake_value = kwargs.pop("fake_value", "fake_token_value") + token = sas_func(*sas_func_pos_args, **kwargs) + + fake_token = self._create_fake_token(token, fake_value) + + if self.is_live: + return token + return fake_token + + def _create_fake_token(self, token, fake_value): + parts = token.split("&") + + for idx, part in enumerate(parts): + if part.startswith("sig"): + key = part.split("=") + key[1] = fake_value + parts[idx] = "=".join(key) + elif part.startswith("st"): + key = part.split("=") + key[1] = "start" + parts[idx] = "=".join(key) + elif part.startswith("se"): + key = part.split("=") + key[1] = "end" + parts[idx] = "=".join(key) + + return "&".join(parts) diff --git a/tools/azure-sdk-tools/devtools_testutils/config.py b/tools/azure-sdk-tools/devtools_testutils/config.py index 117c3b38ca08..7b9bcfbb06aa 100644 --- a/tools/azure-sdk-tools/devtools_testutils/config.py +++ b/tools/azure-sdk-tools/devtools_testutils/config.py @@ -1 +1,9 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + + +PROXY_URL = "https://localhost:5001" TEST_SETTING_FILENAME = "testsettings_local.cfg" diff --git a/tools/azure-sdk-tools/devtools_testutils/enums.py b/tools/azure-sdk-tools/devtools_testutils/enums.py new file mode 100644 index 000000000000..f1456dbc8a1f --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/enums.py @@ -0,0 +1,11 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from enum import Enum + +class ProxyRecordingSanitizer(str, Enum): + """General-purpose sanitizers for sanitizing test proxy recordings""" + + URI = "UriRegexSanitizer" diff --git a/tools/azure-sdk-tools/devtools_testutils/helpers.py b/tools/azure-sdk-tools/devtools_testutils/helpers.py index e751e09cbd55..4d0867aea542 100644 --- a/tools/azure-sdk-tools/devtools_testutils/helpers.py +++ b/tools/azure-sdk-tools/devtools_testutils/helpers.py @@ -3,6 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- + + class RetryCounter(object): def __init__(self): self.count = 0 diff --git a/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py b/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py index 562bb825bec6..d7df0c5422cc 100644 --- a/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py +++ b/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py @@ -65,9 +65,11 @@ def create_resource(self, name, **kwargs): scrubbed_value = self.fake_values[key] if scrubbed_value: self.real_values[key.lower()] = os.environ[key.upper()] - self.test_class_instance.scrubber.register_name_pair( - self.real_values[key.lower()], scrubbed_value - ) + # Adding this for new proxy testcase + if hasattr(self.test_class_instance, "scrubber"): + self.test_class_instance.scrubber.register_name_pair( + self.real_values[key.lower()], scrubbed_value + ) else: template = 'To pass a live ID you must provide the scrubbed value for recordings to \ prevent secrets from being written to files. {} was not given. For example: \ diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py new file mode 100644 index 000000000000..c2de8b0441ec --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -0,0 +1,141 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import os +import requests +import pdb + +try: + # py3 + import urllib.parse as url_parse +except: + # py2 + import urlparse as url_parse + +import subprocess + +# the functions we patch +from azure.core.pipeline.transport import RequestsTransport + +# the trimming function to clean up incoming arguments to the test function we are wrapping +from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function +from .azure_recorded_testcase import is_live +from .config import PROXY_URL + + +# defaults +RECORDING_START_URL = "{}/record/start".format(PROXY_URL) +RECORDING_STOP_URL = "{}/record/stop".format(PROXY_URL) +PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL) +PLAYBACK_STOP_URL = "{}/playback/stop".format(PROXY_URL) + +# TODO, create a pytest scope="session" implementation that can be added to a fixture such that unit tests can +# startup/shutdown the local test proxy +# this should also fire the admin mapping updates, and start/end the session for commiting recording updates + + +def get_test_id(): + # pytest sets the current running test in an environment variable + setting_value = os.getenv("PYTEST_CURRENT_TEST") + + path_to_test = os.path.normpath(setting_value.split(" ")[0]) + path_components = path_to_test.split(os.sep) + + for idx, val in enumerate(path_components): + if val.startswith("test"): + path_components.insert(idx + 1, "recordings") + break + + return os.sep.join(path_components).replace("::", "").replace("\\", "/") + + +def get_current_sha(): + result = subprocess.check_output(["git", "rev-parse", "HEAD"]) + + # TODO: is this compatible with py27? + return result.decode("utf-8").strip() + + +def start_record_or_playback(test_id): + if is_live(): + result = requests.post( + RECORDING_START_URL, + headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, + ) + recording_id = result.headers["x-recording-id"] + else: + result = requests.post( + PLAYBACK_START_URL, + headers={"x-recording-file": test_id, "x-recording-sha": get_current_sha()}, + ) + recording_id = result.headers["x-recording-id"] + return recording_id + + +def stop_record_or_playback(test_id, recording_id): + if is_live(): + requests.post( + RECORDING_STOP_URL, + headers={"x-recording-file": test_id, "x-recording-id": recording_id, "x-recording-save": "true"}, + ) + else: + requests.post( + PLAYBACK_STOP_URL, + headers={"x-recording-file": test_id, "x-recording-id": recording_id}, + ) + + +def get_proxy_netloc(): + parsed_result = url_parse.urlparse(PROXY_URL) + return {"scheme": parsed_result.scheme, "netloc": parsed_result.netloc} + + +def transform_request(request, recording_id): + """Redirect the request to the test proxy, and store the original request URI in a header""" + headers = request.headers + + parsed_result = url_parse.urlparse(request.url) + updated_target = parsed_result._replace(**get_proxy_netloc()).geturl() + if headers.get("x-recording-upstream-base-uri", None) is None: + headers["x-recording-upstream-base-uri"] = "{}://{}".format(parsed_result.scheme, parsed_result.netloc) + headers["x-recording-id"] = recording_id + headers["x-recording-mode"] = "record" if is_live() else "playback" + request.url = updated_target + + +def RecordedByProxy(func): + def record_wrap(*args, **kwargs): + test_id = get_test_id() + recording_id = start_record_or_playback(test_id) + + def transform_args(*args, **kwargs): + copied_positional_args = list(args) + request = copied_positional_args[1] + + transform_request(request, recording_id) + + return tuple(copied_positional_args), kwargs + + trimmed_kwargs = {k: v for k, v in kwargs.items()} + trim_kwargs_from_test_function(func, trimmed_kwargs) + + original_transport_func = RequestsTransport.send + + def combined_call(*args, **kwargs): + adjusted_args, adjusted_kwargs = transform_args(*args, **kwargs) + return original_transport_func(*adjusted_args, **adjusted_kwargs) + + RequestsTransport.send = combined_call + + # call the modified function. + try: + value = func(*args, **trimmed_kwargs) + finally: + RequestsTransport.send = original_transport_func + stop_record_or_playback(test_id, recording_id) + + return value + + return record_wrap