Integrate proxy

.gitignore

@@ -114,3 +114,6 @@ sdk/cosmos/azure-cosmos/test/test_config.py
 
 # env vars
 .env
+
+# local SSL certificate folder
+.certificate

eng/common/scripts/common.ps1

@@ -44,3 +44,4 @@ $GetDocsMsMetadataForPackageFn = "Get-${Language}-DocsMsMetadataForPackage"
 $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageSpecificPackageInfo"
 $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
 $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"
+$TestProxyTrustCertFn = "Import-Dev-Cert-${Language}"

eng/common/scripts/trust-proxy-certificate.ps1

@@ -0,0 +1,6 @@
+. $PSScriptRoot/common.ps1
+
+if ($TestProxyTrustCertFn -and (Test-Path "Function:$TestProxyTrustCertFn"))
+{
+    &$TestProxyTrustCertFn
+}

eng/common/testproxy/docker-start-proxy.ps1

@@ -0,0 +1,83 @@
+ #!/usr/bin/env pwsh -c
+
+<#
+.DESCRIPTION
+Start the docker proxy container. If it is already running, quietly exit. Any other error should fail.
+.PARAMETER Mode
+"start" or "stop" to start up or stop the test-proxy instance.
+.PARAMETER TargetFolder
+The folder in which context the test proxy will be started. Defaults to current working directory.
+#>
+[CmdletBinding(SupportsShouldProcess = $true)]
+param(
+    [ValidateSet("start", "stop")]
+    [String]
+    $Mode,
+    [String]
+    $TargetFolder = "."
+)
+
+try {
+    docker --version | Out-Null
+}
+catch {
+    Write-Error "A invocation of docker --version failed. This indicates that docker is not properly installed or running."
+    Write-Error "Please check your docker invocation and try running the script again."
+}
+
+$SELECTED_IMAGE_TAG = "1037115"
+$CONTAINER_NAME = "ambitious_azsdk_test_proxy"
+$LINUX_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-lin:${SELECTED_IMAGE_TAG}"
+$WINDOWS_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-win:${SELECTED_IMAGE_TAG}"
+$root = (Resolve-Path $TargetFolder).Path.Replace("`\", "/")
+
+function Get-Proxy-Container(){
+    return (docker container ls -a --format "{{ json . }}" --filter "name=$CONTAINER_NAME" `
+                | ConvertFrom-Json `
+                | Select-Object -First 1)
+}
+
+
+$SelectedImage = $LINUX_IMAGE_SOURCE
+$Initial = ""
+
+# most of the time, running this script on a windows machine will work just fine, as docker defaults to linux containers
+# however, in CI, windows images default to _windows_ containers. We cannot swap them. We can tell if we're in a CI build by 
+# checking for the environment variable TF_BUILD.
+if ($IsWindows -and $env:TF_BUILD){
+    $SelectedImage = $WINDOWS_IMAGE_SOURCE
+    $Initial = "C:"
+}
+
+if ($Mode -eq "start"){
+    $proxyContainer = Get-Proxy-Container
+
+    # if we already have one, we just need to check the state
+    if($proxyContainer){
+        if ($proxyContainer.State -eq "running")
+        {
+            Write-Host "Discovered an already running instance of the test-proxy!. Exiting"
+            exit(0)
+        }
+    }
+    # else we need to create it
+    else {
+        Write-Host "Attempting creation of Docker host $CONTAINER_NAME"
+        Write-Host "docker container create -v `"${root}:${Initial}/etc/testproxy`" -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage"
+        docker container create -v "${root}:${Initial}/etc/testproxy" -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage
+    }
+
+    Write-Host "Attempting start of Docker host $CONTAINER_NAME"
+    docker container start $CONTAINER_NAME
+}
+
+if ($Mode -eq "stop"){
+    $proxyContainer = Get-Proxy-Container
+
+    if($proxyContainer){
+        if($proxyContainer.State -eq "running"){
+            Write-Host "Found a running instance of $CONTAINER_NAME, shutting it down."
+            docker container stop $CONTAINER_NAME
+        }
+    }
+}

eng/common/testproxy/dotnet-devcert.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSDCCAjCgAwIBAgIUPMKpJ/j10eQrcQBNnkImIaOYHakwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIxMDgwNTAwMzU1NloXDTIyMDgw
+NTAwMzU1NlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAxe/ZseXgOTVoF7uTjX5Leknk95jIoyGc+VlxA8BhzGOr
+r4u6VNQZRCMq+svHY36tW4+u/xHNe2kvbwy2mnS8cFFLfst+94qBZVJDBxSGZ9I/
+wekErNsjFsik4UrMvcC+ZlGPh7hb3f7tSx29tn1DIkAUXVnbZ6TT5s+mYRQpZ6fW
+6kR3RNfc0A1IUM7Zs9yfNEr0O2H41P2HcLKoOPtvd7GvTQm9Ofh3srKvII+sZn/J
+WH7r76oRQMX904mOMdryQwZLObsqX4dXIEbafKVSecB3PBVIhv8gVtJhcZbQP1pI
+mMiWd6PHv46ZhGf7+cKnYUSa8Ia2t/wetK1wd00dFwIDAQABo4GRMIGOMA8GA1Ud
+EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGmMBYGA1UdJQEB/wQMMAoGCCsGAQUF
+BwMBMBcGA1UdEQEB/wQNMAuCCWxvY2FsaG9zdDA6BgorBgEEAYI3VAEBBCwMKkFT
+UC5ORVQgQ29yZSBIVFRQUyBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTANBgkqhkiG
+9w0BAQsFAAOCAQEAIj2VlBVcXGSly6KCBg6lgwFi+henWfSox77iuGAaAxDjN3jd
+9lZahW4MPNLHKSrPRb4YNSLZ2jh7zdcttQrqd4qH65o1q56q5JrCmli99iIzY9Y8
+RdYyxK4Zzr31wjpsyFiWQfqJTuSFUUg9uDDj0negwEZLIGlt7nr12wflt2+QOJtD
+byMeSZLbB5dPzn341DK0qfJEJMMgL0XsPEVZ3TQ6Alc9zq5wI608C/mXnz3xJE05
+UTYD8pRJJ/DyG0empvOVE8Sg93msHPquAbgqO9aqCpykgg/a8CFvI4wRdfvGEFlv
+8XJKL8Y/PFsmFeO3axq3zUYKFVdc9Un4dFIaag==
+-----END CERTIFICATE-----

eng/common/testproxy/invoke-proxy.sh

@@ -0,0 +1 @@
+${TEST_PROXY_LOC}/test-proxy 

eng/common/testproxy/test-proxy-docker.yml

@@ -0,0 +1,15 @@
+parameters:
+    rootFolder: '$(Build.SourcesDirectory)'
+
+steps:
+  - pwsh: |
+        $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1
+    displayName: 'Language Specific Certificate Trust'
+
+  - pwsh: |
+      $(Build.SourcesDirectory)/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}"
+    displayName: 'Run the docker container'
+
+  - pwsh: |
+      docker container ls -a
+    displayName: Check running container

eng/common/testproxy/test-proxy-tool.yml

@@ -0,0 +1,47 @@
+parameters:
+  rootFolder: '$(Build.SourcesDirectory)'
+
+steps:
+  - pwsh: |
+        $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1
+    displayName: 'Language Specific Certificate Trust'
+
+  - pwsh: |
+      Write-Host "##vso[task.setvariable variable=OriginalPath]$env:PATH"
+    displayName: 'Store Path Value'
+
+  - pwsh: |
+      Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]$(Build.SourcesDirectory)/eng/common/testproxy/dotnet-devcert.pfx"
+      Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password"
+    displayName: 'Configure Kestrel Environment Variables'
+
+  - task: UseDotNet@2
+    displayName: "Use .NET Core SDK"
+    inputs:
+      packageType: sdk
+      version: 5.0.205
+
+  - pwsh: |
+      dotnet tool install azure.sdk.tools.testproxy `
+        --tool-path $(Build.BinariesDirectory)/test-proxy `
+        --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json `
+        --version 1.0.0-dev.20210811.2
+    displayName: "Install test-proxy"
+
+  - pwsh: |
+      Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe `
+        -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" `
+        -NoNewWindow -PassThru
+    displayName: 'Run the testproxy - windows'
+    condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'))
+
+  # nohup does NOT continue beyond the current session if you use it within powershell
+  - bash: |
+      sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy &
+    displayName: "Run the testproxy - linux/mac"
+    condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'))
+    workingDirectory: "${{ parameters.rootFolder }}"
+
+  - pwsh: |
+      Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)"
+    displayName: 'Restore .NET version by resetting path'

eng/pipelines/templates/steps/check-availability.yml

@@ -0,0 +1,4 @@
+steps:
+  - pwsh: |
+      Invoke-WebRequest -Uri "http://localhost:5000/Info/Available"
+    displayName: 'http://localhost:5000/Info/Available'

eng/pipelines/test-with-proxy.yml

@@ -0,0 +1,53 @@
+trigger: none
+
+variables:
+  PythonVersion: '3.9'
+
+jobs:
+  - job: 'win2019docker'
+    displayName: 'Windows Docker Run'
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: UsePythonVersion@0
+        displayName: 'Use Python $(PythonVersion)'
+        inputs:
+          versionSpec: $(PythonVersion)
+      - template: /eng/common/testproxy/test-proxy-docker.yml
+      - template : /eng/pipelines/templates/steps/check-availability.yml
+
+  - job: 'win2019tool'
+    displayName: 'Windows Tool Run'
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: UsePythonVersion@0
+        displayName: 'Use Python $(PythonVersion)'
+        inputs:
+          versionSpec: $(PythonVersion)
+      - template: /eng/common/testproxy/test-proxy-tool.yml
+      - template : /eng/pipelines/templates/steps/check-availability.yml
+
+  - job: 'linux2020docker'
+    displayName: 'Linux Docker Run'
+    pool:
+      vmImage: 'ubuntu-20.04'
+    steps:
+      - task: UsePythonVersion@0
+        displayName: 'Use Python $(PythonVersion)'
+        inputs:
+          versionSpec: $(PythonVersion)
+      - template: /eng/common/testproxy/test-proxy-docker.yml
+      - template : /eng/pipelines/templates/steps/check-availability.yml
+
+  - job: 'linux2020tool'
+    displayName: 'Linux Tool Run'
+    pool:
+      vmImage: 'ubuntu-20.04'
+    steps:
+      - task: UsePythonVersion@0
+        displayName: 'Use Python $(PythonVersion)'
+        inputs:
+          versionSpec: $(PythonVersion)
+      - template: /eng/common/testproxy/test-proxy-tool.yml
+      - template : /eng/pipelines/templates/steps/check-availability.yml

eng/scripts/Language-Settings.ps1

@@ -282,3 +282,7 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null)
     return $null
   }
 }
+function Import-Dev-Cert-python
+{
+  Write-Host "Certificate action here!"
+}

scripts/install_certificate.py

@@ -0,0 +1,33 @@
+import requests
+import os
+
+root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", ".."))
+CERT_URL = "https://raw.githubusercontent.com/Azure/azure-sdk-tools/main/tools/test-proxy/docker/dev_certificate/dotnet-devcert.crt"
+EXISTING_ROOT_PEM = requests.certs.where()
+LOCAL_FILENAME = CERT_URL.split('/')[-1].split(".")[0] + ".pem"
+LOCAL_LOCATION = os.path.join(root_dir, '.certificate', LOCAL_FILENAME)
+
+def download_cert_file():
+    content = requests.get(CERT_URL)
+
+    if content.status_code == 200:
+        with open(LOCAL_LOCATION, 'w') as f:
+            f.write(content.text)
+
+def combine_cert_file():
+    with open(EXISTING_ROOT_PEM, 'r') as f:
+        content = f.readlines();
+
+    with open(LOCAL_LOCATION, 'a') as f:
+        f.writelines(content)
+
+if __name__ == "__main__":
+    download_cert_file()
+    combine_cert_file()
+
+    print("Set the following certificate paths:")
+    print("\tSSL_CERT_DIR={}".format(os.path.dirname(LOCAL_LOCATION)))
+    print("\tREQUESTS_CA_BUNDLE={}".format(LOCAL_LOCATION))
+
+    print("\nBe aware that REQUESTS_CA_BUNDLE should only be modified ")
+

sdk/storage/azure-storage-blob/tests/_shared/testcase.py

@@ -9,6 +9,7 @@
 import time
 import os
 import logging
+import pdb
 from devtools_testutils import (
     AzureMgmtTestCase,
     AzureMgmtPreparer,
@@ -32,11 +33,10 @@
 
 import pytest
 
+ENABLE_LOGGING = True
 from devtools_testutils.storage import StorageTestCase
 
 LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s'
-os.environ['AZURE_STORAGE_ACCOUNT_NAME'] = STORAGE_ACCOUNT_NAME
-os.environ['AZURE_STORAGE_ACCOUNT_KEY'] = STORAGE_ACCOUNT_KEY
 os.environ['AZURE_TEST_RUN_LIVE'] = os.environ.get('AZURE_TEST_RUN_LIVE', None) or RUN_IN_LIVE
 os.environ['AZURE_SKIP_LIVE_RECORDING'] = os.environ.get('AZURE_SKIP_LIVE_RECORDING', None) or SKIP_LIVE_RECORDING
 
@@ -162,19 +162,22 @@ def storage_account():
 
     try:
         if i_need_to_create_rg:
+            logging.info("I NEED TO CREATE RG")
             rg_name, rg_kwargs = rg_preparer._prepare_create_resource(test_case)
             rg = rg_kwargs['resource_group']
         else:
+            logging.info("I DONT NEED TO CREATE RG")
             rg_name = existing_rg_name or "no_rg_needed"
             rg = FakeResource(
                 name=rg_name,
                 id="/subscriptions/{}/resourceGroups/{}".format(subscription_id, rg_name)
             )
         StorageTestCase._RESOURCE_GROUP = rg
 
+
         try:
             if got_storage_info_from_env:
-
+                logging.info("EXISTING INFO IN ENV")
                 if storage_connection_string:
                     storage_connection_string_parts = dict([
                         part.split('=', 1)
@@ -183,6 +186,7 @@ def storage_account():
 
                 storage_account = None
                 if existing_storage_name:
+                    logging.info("EXISTING NAME IS {}".format(existing_storage_name))
                     storage_name = existing_storage_name
                     storage_account = StorageAccount(
                         location=location,

sdk/storage/azure-storage-blob/tests/test_blob_client.py

@@ -15,7 +15,7 @@
     ContainerClient,
     BlobClient,
 )
-from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer
+from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer, RecordedByProxy
 from _shared.testcase import GlobalStorageAccountPreparer
 from devtools_testutils.storage import StorageTestCase
 
@@ -48,9 +48,9 @@ def validate_standard_account_endpoints(self, service, url_type, name, storage_a
 
     # --Direct Parameters Test Cases --------------------------------------------
     @GlobalStorageAccountPreparer()
+    @RecordedByProxy
     def test_create_service_with_key(self, resource_group, location, storage_account, storage_account_key):
         # Arrange
-
         for client, url in SERVICES.items():
             # Act
             service = client(

sdk/tables/azure-data-tables/tests/async_preparers.py

@@ -1,6 +1,7 @@
 import time
 
 from azure.core.credentials import AzureNamedKeyCredential
+from devtools_testutils.aio import RecordedByProxyAsync
 from azure.core.exceptions import HttpResponseError
 
 from devtools_testutils import is_live
@@ -10,6 +11,7 @@
 
 def cosmos_decorator_async(func, **kwargs):
     @CosmosPreparer()
+    @RecordedByProxyAsync
     async def wrapper(*args, **kwargs):
         key = kwargs.pop("tables_primary_cosmos_account_key")
         name = kwargs.pop("tables_cosmos_account_name")
@@ -28,6 +30,7 @@ async def wrapper(*args, **kwargs):
 
 def tables_decorator_async(func, **kwargs):
     @TablesPreparer()
+    @RecordedByProxyAsync
     async def wrapper(*args, **kwargs):
         key = kwargs.pop("tables_primary_storage_account_key")
         name = kwargs.pop("tables_storage_account_name")