diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 133137ea79aa..e26d1515933a 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -62,7 +62,10 @@ param ( [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID), [Parameter()] - [switch] $Force + [switch] $Force, + + [Parameter()] + [switch] $OutFile ) # By default stop for any error. @@ -108,7 +111,8 @@ trap { } # Enumerate test resources to deploy. Fail if none found. -$root = [System.IO.Path]::Combine("$PSScriptRoot/../../../sdk", $ServiceDirectory) | Resolve-Path +$repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path +$root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path $templateFileName = 'test-resources.json' $templateFiles = @() @@ -172,19 +176,20 @@ if ($TestApplicationId -and !$TestApplicationOid) { } } + +# If the ServiceDirectory is an absolute path use the last directory name +# (e.g. D:\foo\bar\ -> bar) +$serviceName = if (Split-Path -IsAbsolute $ServiceDirectory) { + Split-Path -Leaf $ServiceDirectory +} else { + $ServiceDirectory +} + # Format the resource group name based on resource group naming recommendations and limitations. $resourceGroupName = if ($CI) { $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16) Write-Verbose "Generated base name '$BaseName' for CI build" - # If the ServiceDirectory is an absolute path use the last directory name - # (e.g. D:\foo\bar\ -> bar) - $serviceName = if (Split-Path -IsAbsolute $ServiceDirectory) { - Split-Path -Leaf $ServiceDirectory - } else { - $ServiceDirectory - } - "rg-{0}-$BaseName" -f ($serviceName -replace '[\\\/:]', '-').Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-') } else { "rg-$BaseName" @@ -281,12 +286,21 @@ foreach ($templateFile in $templateFiles) { Write-Verbose "Successfully deployed template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'" } - if ($deployment.Outputs.Count -and !$CI) { - # Write an extra new line to isolate the environment variables for easy reading. - Log "Persist the following environment variables based on your detected shell ($shell):`n" + $serviceDirectoryPrefix = $serviceName.ToUpperInvariant() + "_" + + $context = Get-AzContext; + + # Add default values + $deploymentOutputs = @{ + "$($serviceDirectoryPrefix)CLIENT_ID" = $TestApplicationId; + "$($serviceDirectoryPrefix)CLIENT_SECRET" = $TestApplicationSecret; + "$($serviceDirectoryPrefix)TENANT_ID" = $context.Tenant.Id; + "$($serviceDirectoryPrefix)SUBSCRIPTION_ID" = $context.Subscription.Id; + "$($serviceDirectoryPrefix)RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; + "$($serviceDirectoryPrefix)LOCATION" = $resourceGroup.Location; + "$($serviceDirectoryPrefix)ENVIRONMENT" = $context.Environment.Name; } - $deploymentOutputs = @{} foreach ($key in $deployment.Outputs.Keys) { $variable = $deployment.Outputs[$key] @@ -295,23 +309,54 @@ foreach ($templateFile in $templateFiles) { if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { $deploymentOutputs[$key] = $variable.Value + } + } + if ($OutFile) + { + if (!$IsWindows) + { + Write-Host "File option is supported only on Windows" + } + + $outputFile = "$templateFile.env" + + $environmentText = $deploymentOutputs | ConvertTo-Json; + $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText) + $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + + Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + + Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" + } + else + { + + if (!$CI) { + # Write an extra new line to isolate the environment variables for easy reading. + Log "Persist the following environment variables based on your detected shell ($shell):`n" + } + + foreach ($key in $deploymentOutputs.Keys) + { + $value = $deploymentOutputs[$key] + if ($CI) { # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. Write-Host "Setting variable '$key': ***" - Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($variable.Value)" - Write-Host "##vso[task.setvariable variable=$key;]$($variable.Value)" + Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($value)" + Write-Host "##vso[task.setvariable variable=$key;]$($value)" } else { - Write-Host ($shellExportFormat -f $key, $variable.Value) + Write-Host ($shellExportFormat -f $key, $value) } } - } - if ($key) { - # Isolate the environment variables for easy reading. - Write-Host "`n" - $key = $null + if ($key) { + # Isolate the environment variables for easy reading. + Write-Host "`n" + $key = $null + } } $postDeploymentScript = $templateFile | Split-Path | Join-Path -ChildPath 'test-resources-post.ps1' @@ -442,6 +487,10 @@ Deployment (CI/CD) build (only Azure Pipelines is currently supported). .PARAMETER Force Force creation of resources instead of being prompted. +.PARAMETER OutFile +Save test environment settings into a test-resources.json.env file next to test-resources.json. File is protected via DPAPI. Supported only on windows. +The environment file would be scoped to the current repository directory. + .EXAMPLE Connect-AzAccount -Subscription "REPLACE_WITH_SUBSCRIPTION_ID" $testAadApp = New-AzADServicePrincipal -Role Owner -DisplayName 'azure-sdk-live-test-app' diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index 961452fe8ca1..53debad343e1 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -38,6 +38,9 @@ param ( [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)] [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)] [string] $ProvisionerApplicationSecret, + + [Parameter()] + [string] $ServiceDirectory, [Parameter()] [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud')] @@ -116,6 +119,15 @@ if (!$ResourceGroupName) { $ResourceGroupName = "rg-$BaseName" } +if (![string]::IsNullOrWhiteSpace($ServiceDirectory)) { + $root = [System.IO.Path]::Combine("$PSScriptRoot/../../../sdk", $ServiceDirectory) | Resolve-Path + $preRemovalScript = Join-Path -Path $root -ChildPath 'remove-test-resources-pre.ps1' + if (Test-Path $preRemovalScript) { + Log "Invoking pre resource removal script '$preRemovalScript'" + &$preRemovalScript -ResourceGroupName $ResourceGroupName @PSBoundParameters + } +} + Log "Deleting resource group '$ResourceGroupName'" if (Retry { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force }) { Write-Verbose "Successfully deleted resource group '$ResourceGroupName'" @@ -157,6 +169,10 @@ A service principal ID to provision test resources when a provisioner is specifi .PARAMETER ProvisionerApplicationSecret A service principal secret (password) to provision test resources when a provisioner is specified. +.PARAMETER ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover pre removal script named 'remove-test-resources-pre.json'. + .PARAMETER Environment Name of the cloud environment. The default is the Azure Public Cloud ('PublicCloud') diff --git a/eng/common/TestResources/Remove-TestResources.ps1.md b/eng/common/TestResources/Remove-TestResources.ps1.md index edf3990ce93b..f9bc1803ae32 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1.md +++ b/eng/common/TestResources/Remove-TestResources.ps1.md @@ -169,6 +169,22 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - specified - in which to discover pre removal script named 'remove-test-resources-pre.json'. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -Environment Name of the cloud environment. The default is the Azure Public Cloud diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index 40eec569fa37..e79dfe1267e6 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -7,6 +7,9 @@ # Assumes steps in deploy-test-resources.yml was run previously. Requires # environment variable: AZURE_RESOURCEGROUP_NAME and Az PowerShell module +parameters: + ServiceDirectory: '' + steps: - pwsh: > eng/common/TestResources/Remove-TestResources.ps1 @@ -15,6 +18,7 @@ steps: -SubscriptionId '$(azure-subscription-id)' -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id)' -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret)' + -ServiceDirectory '${{ parameters.ServiceDirectory }}' -Environment 'AzureCloud' -Force -Verbose @@ -29,6 +33,7 @@ steps: -SubscriptionId '$(azure-subscription-id-gov)' -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-gov)' -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-gov)' + -ServiceDirectory '${{ parameters.ServiceDirectory }}' -Environment 'AzureUSGovernment' -Force -Verbose @@ -43,6 +48,7 @@ steps: -SubscriptionId '$(azure-subscription-id-cn)' -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-cn)' -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-cn)' + -ServiceDirectory '${{ parameters.ServiceDirectory }}' -Environment 'AzureChinaCloud' -Force -Verbose diff --git a/eng/common/pipelines/templates/steps/bypass-local-dns.yml b/eng/common/pipelines/templates/steps/bypass-local-dns.yml new file mode 100644 index 000000000000..54f49c33714b --- /dev/null +++ b/eng/common/pipelines/templates/steps/bypass-local-dns.yml @@ -0,0 +1,6 @@ + +steps: + # https://github.com/actions/virtual-environments/issues/798 + - script: sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf + displayName: Bypass local DNS server to workaround issue resolving cognitiveservices names + condition: and(succeededOrFailed(), eq(variables['OSVmImage'], 'ubuntu-18.04')) \ No newline at end of file diff --git a/eng/common/scripts/copy-readmes-to-docs.ps1 b/eng/common/scripts/copy-readmes-to-docs.ps1 new file mode 100644 index 000000000000..285eefce5a18 --- /dev/null +++ b/eng/common/scripts/copy-readmes-to-docs.ps1 @@ -0,0 +1,85 @@ +# Example Usage: ./copy-readmes-to-docs.ps1 -CodeRepo C:/repo/sdk-for-python -DocRepo C:/repo/azure-docs-sdk-python +# run the link updating script before this to get links fixed to the release +# Highly recommended that you use Powershell Core. +# git reset --hard origin/smoke-test on the doc repo prior to running this + +param ( + [String]$CodeRepo, + [String]$DocRepo, + [String]$TargetServices +) + +$PACKAGE_README_REGEX = ".*[\/\\]sdk[\\\/][^\/\\]+[\\\/][^\/\\]+[\/\\]README\.md" + +Write-Host "repo is $CodeRepo" + +$date = Get-Date -Format "MM/dd/yyyy" + +if ($CodeRepo -Match "net") +{ + $lang = ".NET" + $TARGET_FOLDER = Join-Path -Path $DocRepo -ChildPath "api/overview/azure" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv" +} +if ($CodeRepo -Match "python"){ + $lang = "Python" + $TARGET_FOLDER = Join-Path -Path $DocRepo -ChildPath "docs-ref-services" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv" +} +if ($CodeRepo -Match "java"){ + $lang = "Java" + $TARGET_FOLDER = Join-Path -Path $DocRepo -ChildPath "docs-ref-services" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv" +} +if ($CodeRepo -Match "js"){ + $lang = "JavaScript" + $TARGET_FOLDER = Join-Path -Path $DocRepo -ChildPath "docs-ref-services" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv" +} + + + +$metadataResponse = Invoke-WebRequest -Uri $metadataUri | ConvertFrom-Csv + +if ([string]::IsNullOrWhiteSpace($TargetServices)) +{ + $selectedServices = $metadataResponse | ForEach-Object -Process {$_.RepoPath} | Get-Unique +} +else { + $selectedServices = $TargetServices -Split "," | % { return $_.Trim() } +} + + +foreach($service in $selectedServices){ + $readmePath = Join-Path -Path $CodeRepo -ChildPath "sdk/$service" + Write-Host "Examining: $readmePath" + + $libraries = $metadataResponse | Where-Object { $_.RepoPath -eq $service } + + foreach($library in $libraries){ + + $package = $library.Package + $version = If ([string]::IsNullOrWhiteSpace($library.VersionGA)) {$library.VersionPreview} Else {$library.VersionGA} + + $file = Join-Path -Path $readmePath -ChildPath "/$package/README.md" | Get-Item + Write-Host "`tOutputting $($file.FullName)" + + $fileContent = Get-Content $file + + $fileContent = $fileContent -Join "`n" + + $fileMatch = (Select-String -InputObject $fileContent -Pattern 'Azure .+? (client|plugin|shared) library for (JavaScript|Java|Python|\.NET)').Matches[0] + + $header = "---`r`ntitle: $fileMatch`r`nkeywords: Azure, $lang, SDK, API, $service, $package`r`nauthor: maggiepint`r`nms.author: magpint`r`nms.date: $date`r`nms.topic: article`r`nms.prod: azure`r`nms.technology: azure`r`nms.devlang: $lang`r`nms.service: $service`r`n---`r`n" + + $fileContent = $fileContent -replace $fileMatch, "$fileMatch - Version $version `r`n" + + $fileContent = "$header $fileContent" + + $readmeName = "$($file.Directory.Name.Replace('azure-','').Replace('Azure.', '').ToLower())-readme.md" + + $readmeOutputLocation = Join-Path $TARGET_FOLDER -ChildPath $readmeName + + Set-Content -Path $readmeOutputLocation -Value $fileContent + } +} \ No newline at end of file