[KeyVault] KV Certificates Client Library

codegen_to_sdk_config.json

@@ -0,0 +1,30 @@
+{
+    "init": {
+        "initScript": {
+            "path": "./eng/scripts/automation_init.sh",
+            "logPrefix": "[GO]",
+            "stderr":{
+                "storeAllLog": true
+            }
+        }
+    },
+    "generateAndBuild": {
+        "generateAndBuildScript": {
+            "path": "generator automation-v2",
+            "logPrefix": "[GO-Generate]",
+            "stderr":{
+                "storeLogByFilter": "[error|Error|Exception]"
+            }
+        }
+    },
+    "mockTest": {
+        "mockTestScript": {
+            "path": "./eng/scripts/automation/Invoke-MockTest.ps1",
+            "script": "pwsh",
+            "logPrefix": "[GO-MockTest]",
+            "stderr":{
+                "storeLogByFilter": "[error|Error|Exception]"
+            }
+        }
+    }
+}

eng/common/TestResources/New-TestResources.ps1

@@ -45,6 +45,10 @@ param (
     [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')]
     [string] $ProvisionerApplicationId,
 
+    [Parameter(ParameterSetName = 'Provisioner', Mandatory = $false)]
+    [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')]
+    [string] $ProvisionerApplicationOid,
+
     [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)]
     [string] $ProvisionerApplicationSecret,
 
@@ -155,7 +159,7 @@ function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGrou
             $appId = $servicePrincipal.AppId
         } else {
             Write-Verbose "Creating service principal credential via MS Graph API"
-            # In 7.1.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the
+            # In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the
             # parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter.
             $credential = Retry { $servicePrincipal | New-AzADSpCredential }
             $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force
@@ -481,19 +485,19 @@ try {
     $context = Get-AzContext;
 
     # Make sure the provisioner OID is set so we can pass it through to the deployment.
-    $provisionerApplicationOid = if (!$ProvisionerApplicationId) {
+    if (!$ProvisionerApplicationId -and !$ProvisionerApplicationOid) {
         if ($context.Account.Type -eq 'User') {
             $user = Get-AzADUser -UserPrincipalName $context.Account.Id
-            $user.Id
+            $ProvisionerApplicationOid = $user.Id
         } elseif ($context.Account.Type -eq 'ServicePrincipal') {
             $sp = Get-AzADServicePrincipal -ApplicationId $context.Account.Id
-            $sp.Id
+            $ProvisionerApplicationOid = $sp.Id
         } else {
             Write-Warning "Getting the OID for provisioner type '$($context.Account.Type)' is not supported and will not be passed to deployments (seldom required)."
         }
-    } else {
+    } elseif (!$ProvisionerApplicationOid) {
         $sp = Get-AzADServicePrincipal -ApplicationId $ProvisionerApplicationId
-        $sp.Id
+        $ProvisionerApplicationOid = $sp.Id
     }
 
     # If the ServiceDirectory has multiple segments use the last directory name
@@ -651,7 +655,9 @@ try {
         baseName = $BaseName
         testApplicationId = $TestApplicationId
         testApplicationOid = "$TestApplicationOid"
-        provisionerApplicationOid = "$provisionerApplicationOid"
+    }
+    if ($ProvisionerApplicationOid) {
+        $templateParameters["provisionerApplicationOid"] = "$ProvisionerApplicationOid"
     }
 
     if ($TenantId) {

eng/common/docgeneration/Generate-DocIndex.ps1

@@ -119,7 +119,9 @@ function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang, [String]
         $serviceName = $serviceMapping.Value[0]
         $displayName = $serviceMapping.Value[1]
 
-        $fileName = ($serviceName -replace '\s', '').ToLower().Trim()
+        # handle spaces in service name, EG "Confidential Ledger"
+        # handle / in service name, EG "Database for MySQL/PostgreSQL". Leaving a "/" present will generate a bad link location.
+        $fileName = ($serviceName -replace '\s', '').Replace("/","").ToLower().Trim()
         if ($visitedService.ContainsKey($serviceName)) {
             if ($displayName) {
                 Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact`n##### ($displayName)"

eng/common/pipelines/templates/steps/retain-run.yml

@@ -1,6 +1,6 @@
 parameters:
   - name: DaysValid
-    default: 365
+    default: 731
     type: number
 
 steps:
@@ -16,7 +16,6 @@ steps:
         -Project $(System.TeamProject)
         -DefinitionId $(System.DefinitionId)
         -RunId $(Build.BuildId)
-        -OwnerId Pipeline
-        -DaysValid ${{parameters.DaysValid}}
+        -DaysValid ${{ parameters.DaysValid }}
         -AccessToken $env:SYSTEM_ACCESSTOKEN
         -Debug

eng/common/pipelines/templates/steps/verify-links.yml

@@ -6,7 +6,7 @@ parameters:
   Recursive: $false
   CheckLinkGuidance: $true
   Urls: '(Get-ChildItem -Path ./ -Recurse -Include *.md)'
-  BranchReplaceRegex: "^(${env:SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI}.*/(?:blob|tree)/)$(DefaultBranch)(/.*)$"
+  BranchReplaceRegex: "^(${env:SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI}/(?:blob|tree)/)$(DefaultBranch)(/.*)$"
   BranchReplacementName: "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}"
   Condition: succeeded() # If you want to run on failure for the link checker, set it to `Condition: succeededOrFailed()`.
 

eng/common/scripts/Add-RetentionLease.ps1

@@ -12,28 +12,33 @@ param(
   [Parameter(Mandatory = $true)]
   [int]$RunId,
 
-  [Parameter(Mandatory = $true)]
-  [string]$OwnerId,
-
   [Parameter(Mandatory = $true)]
   [int]$DaysValid,
 
-  [Parameter(Mandatory = $true)]
-  [string]$AccessToken
+  [Parameter(Mandatory = $false)]
+  [string]$OwnerId = "azure-sdk-pipeline-automation",
+
+  [Parameter(Mandatory = $false)]
+  [string]$AccessToken = $env:DEVOPS_PAT
 )
 
+Set-StrictMode -Version 3
+
+. (Join-Path $PSScriptRoot common.ps1)
+
 $unencodedAuthToken = "nobody:$AccessToken"
 $unencodedAuthTokenBytes = [System.Text.Encoding]::UTF8.GetBytes($unencodedAuthToken)
 $encodedAuthToken = [System.Convert]::ToBase64String($unencodedAuthTokenBytes)
 
-# We are doing this here so that there is zero chance that this token is emitted in Azure Pipelines
-# build logs. Azure Pipelines will see this text and register the secret as a value it should *** out
-# before being transmitted to the server (and shown in logs). It means if the value is accidentally
-# leaked anywhere else that it won't be visible. The downside is that when the script is executed
-# on a local development box, it will be visible.
-Write-Host "##vso[task.setvariable variable=_throwawayencodedaccesstoken;issecret=true;]$($encodedAuthToken)"
+if ($isDevOpsRun) {
+  # We are doing this here so that there is zero chance that this token is emitted in Azure Pipelines
+  # build logs. Azure Pipelines will see this text and register the secret as a value it should *** out
+  # before being transmitted to the server (and shown in logs). It means if the value is accidentally
+  # leaked anywhere else that it won't be visible. The downside is that when the script is executed
+  # on a local development box, it will be visible.
+  Write-Host "##vso[task.setvariable variable=_throwawayencodedaccesstoken;issecret=true;]$($encodedAuthToken)"
+}
 
-. (Join-Path $PSScriptRoot common.ps1)
 
 LogDebug "Checking for existing leases on run: $RunId"
 $existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedAuthToken $encodedAuthToken

eng/common/scripts/Get-AADIdentityFromGithubUser.ps1

@@ -0,0 +1,81 @@
+<#
+.DESCRIPTION
+Get the corresponding ms alias from github identity
+.PARAMETER AadToken
+The aad access token.
+.PARAMETER GithubName
+Github identity. E.g sima-zhu
+.PARAMETER ContentType
+Content type of http requests.
+.PARAMETER AdditionalHeaders
+Additional parameters for http request headers in key-value pair format, e.g. @{ key1 = val1; key2 = val2; key3 = val3}
+#>
+[CmdletBinding(SupportsShouldProcess = $true)]
+param(
+  [Parameter(Mandatory = $true)]
+  [string]$TenantId,
+
+  [Parameter(Mandatory = $true)]
+  [string]$ClientId,
+
+  [Parameter(Mandatory = $true)]
+  [string]$ClientSecret,
+
+  [Parameter(Mandatory = $true)]
+  [string]$GithubUser
+)
+Set-StrictMode -Version 3
+
+. "${PSScriptRoot}\common.ps1"
+
+$OpensourceAPIBaseURI = "https://repos.opensource.microsoft.com/api/people/links/github/$GithubUser"
+
+function Generate-AadToken ($TenantId, $ClientId, $ClientSecret) {
+    $LoginAPIBaseURI = "https://login.microsoftonline.com/$TenantId/oauth2/token"
+    try {
+        $headers = @{
+            "content-type" = "application/x-www-form-urlencoded"
+        }
+
+        $body = @{
+            "grant_type" = "client_credentials"
+            "client_id" = $ClientId
+            "client_secret" = $ClientSecret
+            "resource" = "api://repos.opensource.microsoft.com/audience/7e04aa67"
+        }
+        Write-Host "Generating aad token..."
+        $resp = Invoke-RestMethod $LoginAPIBaseURI -Method 'POST' -Headers $headers -Body $body
+    }
+    catch { 
+        LogError $_
+        exit 1
+    }
+
+    return $resp.access_token
+} 
+
+$Headers = @{
+    "Content-Type" = "application/json"
+    "api-version" = "2019-10-01"
+}
+
+try {
+    $opsAuthToken = Generate-AadToken -TenantId $TenantId -ClientId $ClientId -ClientSecret $ClientSecret
+    $Headers["Authorization"] = "Bearer $opsAuthToken"
+    Write-Host "Fetching aad identity for github user: $GithubName"
+    $resp = Invoke-RestMethod $OpensourceAPIBaseURI -Method 'GET' -Headers $Headers
+}
+catch { 
+    LogError $_
+    exit 1
+}
+
+$resp | Write-Verbose
+
+if ($resp.aad) {
+    Write-Host "Fetched aad identity $($resp.aad.alias) for github user $GithubName."
+    return $resp.aad.alias
+}
+
+LogError "Failed to retrieve the aad identity from given github user: $GithubName"
+exit 1

eng/common/scripts/Invoke-DevOpsAPI.ps1

@@ -137,7 +137,7 @@ function Add-RetentionLease {
     $RunId,
     $OwnerId,
     $DaysValid,
-    $Base64AuthToken
+    $Base64EncodedAuthToken
   )
 
   $parameter = @{}

eng/common/scripts/stress-testing/find-all-stress-packages.ps1

@@ -7,6 +7,8 @@ class StressTestPackageInfo {
     [string]$Namespace
     [string]$Directory
     [string]$ReleaseName
+    [string]$Dockerfile
+    [string]$DockerBuildDir
 }
 
 function FindStressPackages([string]$directory, [hashtable]$filters = @{}, [switch]$CI) {
@@ -52,6 +54,8 @@ function NewStressTestPackageInfo([hashtable]$chart, [System.IO.FileInfo]$chartF
         Namespace = $namespace.ToLower()
         Directory = $chartFile.DirectoryName
         ReleaseName = $chart.name
+        Dockerfile = $chart.annotations.dockerfile
+        DockerBuildDir = $chart.annotations.dockerbuilddir
     }
 }
 

eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1

@@ -86,12 +86,6 @@ function DeployStressTests(
         $subscription = 'Azure SDK Test Resources'
     }
 
-    if (!$repository) {
-        $repository = if ($env:USER) { $env:USER } else { "${env:USERNAME}" }
-        # Remove spaces, etc. that may be in $namespace
-        $repository -replace '\W'
-    }
-
     if ($login) {
         if (!$clusterGroup -or !$subscription) {
             throw "clusterGroup and subscription parameters must be specified when logging into an environment that is not test or prod."
@@ -156,11 +150,24 @@ function DeployStressPackage(
     }
     $imageTag += "/$($pkg.Namespace)/$($pkg.ReleaseName):${deployId}"
 
-    $dockerFilePath = "$($pkg.Directory)/Dockerfile"
+    $dockerFilePath = if ($pkg.Dockerfile) {
+        Join-Path $pkg.Directory $pkg.Dockerfile
+    } else {
+        "$($pkg.Directory)/Dockerfile"
+    }
+    $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath)
+
     if ($pushImages -and (Test-Path $dockerFilePath)) {
         Write-Host "Building and pushing stress test docker image '$imageTag'"
         $dockerFile = Get-ChildItem $dockerFilePath
-        Run docker build -t $imageTag -f $dockerFile.FullName $dockerFile.DirectoryName
+        $dockerBuildFolder = if ($pkg.DockerBuildDir) {
+            Join-Path $pkg.Directory $pkg.DockerBuildDir
+        } else {
+            $dockerFile.DirectoryName
+        }
+        $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim()
+
+        Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder
         if ($LASTEXITCODE) { return }
         Run docker push $imageTag
         if ($LASTEXITCODE) {

eng/common/testproxy/docker-start-proxy.ps1

@@ -31,7 +31,7 @@ catch {
     Write-Error "Please check your docker invocation and try running the script again."
 }
 
-$SELECTED_IMAGE_TAG = "1294199"
+$SELECTED_IMAGE_TAG = "1314089"
 $CONTAINER_NAME = "ambitious_azsdk_test_proxy"
 $LINUX_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-lin:${SELECTED_IMAGE_TAG}"
 $WINDOWS_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-win:${SELECTED_IMAGE_TAG}"

eng/common/testproxy/test-proxy-tool.yml

@@ -16,7 +16,7 @@ steps:
       dotnet tool install azure.sdk.tools.testproxy `
         --tool-path $(Build.BinariesDirectory)/test-proxy `
         --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json `
-        --version 1.0.0-dev.20220113.1
+        --version 1.0.0-dev.20220119.2
     displayName: "Install test-proxy"
 
   - pwsh: |

eng/config.json

@@ -16,14 +16,6 @@
             "Name": "azidentity",
             "CoverageGoal": 0.68
         },
-        {
-            "Name": "keyvault/azkeys",
-            "CoverageGoal": 0.71
-        },
-        {
-            "Name": "azsecrets",
-            "CoverageGoal": 0.86
-        },
         {
             "Name": "data",
             "CoverageGoal": 0.62,
@@ -37,6 +29,18 @@
             "Name": "eng/tools",
             "CoverageGoal": 0.0
         },
+        {
+            "Name": "keyvault/azcertificates",
+            "CoverageGoal": 0.78
+        },
+        {
+            "Name": "keyvault/azkeys",
+            "CoverageGoal": 0.71
+        },
+        {
+            "Name": "keyvault/azsecrets",
+            "CoverageGoal": 0.86
+        },
         {
             "Name": "keyvault/internal",
             "CoverageGoal": 0.40

eng/pipelines/templates/steps/build-test.yml

@@ -44,7 +44,7 @@ steps:
     displayName: "Install Coverage and Junit Dependencies"
 
   - ${{ if eq(parameters.TestProxy, true) }}:
-    - template: /eng/common/testproxy/test-proxy-tool.yml
+    - template: /eng/common/testproxy/test-proxy-docker.yml
 
   - task: PowerShell@2
     displayName: 'Run Tests'
@@ -60,8 +60,9 @@ steps:
 
   - ${{ if eq(parameters.TestProxy, true) }}:
     - pwsh: |
-        # $(Build.SourcesDirectory)/test-proxy.log is the hardcoded output log location for the test-proxy-tool.yml
-        cat $(Build.SourcesDirectory)/test-proxy.log
+        # ambitious_azsdk_test_proxy is the hardcoded container name used
+        # by the test proxy startup script
+        docker logs ambitious_azsdk_test_proxy
       displayName: 'Dump Test Proxy logs'
       condition: succeededOrFailed()