Default test recording sanitizers

eng/pipelines/templates/steps/build-test.yml

@@ -64,8 +64,6 @@ steps:
 
   - ${{ if eq(parameters.TestProxy, true) }}:
     - template: /eng/common/testproxy/test-proxy-tool.yml
-      parameters:
-        targetVersion: '1.0.0-dev.20230427.1'
 
   - task: PowerShell@2
     displayName: 'Run Tests'

sdk/internal/CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features Added
 * Options types for `SetBodilessMatcher` and `SetDefaultMatcher` now embed `RecordingOptions`
+* Added a collection of default sanitizers for test recordings
 
 ### Breaking Changes
 

sdk/internal/recording/default_sanitizers.go

@@ -0,0 +1,205 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// this file contains a set of default sanitizers applied to all recordings
+
+package recording
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// sanitizer represents a single sanitizer configured via the test proxy's /Admin/AddSanitizers endpoint
+type sanitizer struct {
+	// Name is the name of a sanitizer type e.g. "BodyKeySanitizer"
+	Name string        `json:"Name,omitempty"`
+	Body sanitizerBody `json:"Body,omitempty"`
+}
+
+type sanitizerBody struct {
+	// GroupForReplace is the name of the regex group to replace
+	GroupForReplace string `json:"groupForReplace,omitempty"`
+	// JSONPath is the JSON path to the value to replace
+	JSONPath string `json:"jsonPath,omitempty"`
+	// Key is the name of a header to sanitize
+	Key string `json:"key,omitempty"`
+	// Regex is the regular expression to match a value to sanitize
+	Regex string `json:"regex,omitempty"`
+	// Value is the string that replaces the matched value. The sanitizers in
+	// this file accept the test proxy's default Value, "Sanitized".
+	Value string `json:"value,omitempty"`
+}
+
+func newBodyKeySanitizer(jsonPath string) sanitizer {
+	return sanitizer{
+		Name: "BodyKeySanitizer",
+		Body: sanitizerBody{
+			JSONPath: jsonPath,
+		},
+	}
+}
+
+func newBodyRegexSanitizer(regex, groupForReplace string) sanitizer {
+	return sanitizer{
+		Name: "BodyRegexSanitizer",
+		Body: sanitizerBody{
+			GroupForReplace: groupForReplace,
+			Regex:           regex,
+		},
+	}
+}
+
+func newGeneralRegexSanitizer(regex, groupForReplace string) sanitizer {
+	return sanitizer{
+		Name: "GeneralRegexSanitizer",
+		Body: sanitizerBody{
+			GroupForReplace: groupForReplace,
+			Regex:           regex,
+		},
+	}
+}
+
+func newHeaderRegexSanitizer(key, regex, groupForReplace string) sanitizer {
+	return sanitizer{
+		Name: "HeaderRegexSanitizer",
+		Body: sanitizerBody{
+			GroupForReplace: groupForReplace,
+			Key:             key,
+			Regex:           regex,
+		},
+	}
+}
+
+// addSanitizers adds an arbitrary number of sanitizers with a single request. It
+// isn't exported because SDK modules don't add enough sanitizers to benefit from it.
+func addSanitizers(s []sanitizer, options *RecordingOptions) error {
+	if options == nil {
+		options = defaultOptions()
+	}
+	url := fmt.Sprintf("%s/Admin/AddSanitizers", options.baseURL())
+	req, err := http.NewRequest(http.MethodPost, url, nil)
+	if err != nil {
+		return err
+	}
+	handleTestLevelSanitizer(req, options)
+	b, err := json.Marshal(s)
+	if err != nil {
+		return err
+	}
+	req.Body = io.NopCloser(bytes.NewReader(b))
+	req.ContentLength = int64(len(b))
+	req.Header.Set("Content-Type", "application/json")
+	return handleProxyResponse(client.Do(req))
+}
+
+var defaultSanitizers = []sanitizer{
+	newGeneralRegexSanitizer(`("|;)Secret=(?<secret>[^;]+)`, "secret"),
+	newBodyKeySanitizer("$..refresh_token"),
+	newHeaderRegexSanitizer("api-key", "", ""),
+	newBodyKeySanitizer("$..access_token"),
+	newBodyKeySanitizer("$..connectionString"),
+	newBodyKeySanitizer("$..applicationSecret"),
+	newBodyKeySanitizer("$..apiKey"),
+	newBodyRegexSanitizer(`client_secret=(?<secret>[^&"]+)`, "secret"),
+	newBodyRegexSanitizer(`client_assertion=(?<secret>[^&"]+)`, "secret"),
+	newHeaderRegexSanitizer("x-ms-rename-source", "", ""),
+	newHeaderRegexSanitizer("x-ms-file-rename-source-authorization", "", ""),
+	newHeaderRegexSanitizer("x-ms-file-rename-source", "", ""),
+	newHeaderRegexSanitizer("x-ms-encryption-key-sha256", "", ""),
+	newHeaderRegexSanitizer("x-ms-encryption-key", "", ""),
+	newHeaderRegexSanitizer("x-ms-copy-source-authorization", "", ""),
+	newHeaderRegexSanitizer("x-ms-copy-source", "", ""),
+	newBodyRegexSanitizer("token=(?<token>[^&]+)($|&)", "token"),
+	newHeaderRegexSanitizer("subscription-key", "", ""),
+	newBodyKeySanitizer("$..sshPassword"),
+	newBodyKeySanitizer("$..secondaryKey"),
+	newBodyKeySanitizer("$..runAsPassword"),
+	newBodyKeySanitizer("$..primaryKey"),
+	newHeaderRegexSanitizer("Location", "", ""),
+	newGeneralRegexSanitizer(`("|;)[Aa]ccess[Kk]ey=(?<secret>[^;]+)`, "secret"),
+	newGeneralRegexSanitizer(`("|;)[Aa]ccount[Kk]ey=(?<secret>[^;]+)`, "secret"),
+	newBodyKeySanitizer("$..aliasSecondaryConnectionString"),
+	newGeneralRegexSanitizer(`("|;)[Ss]hared[Aa]ccess[Kk]ey=(?<secret>[^;\"]+)`, "secret"),
+	newHeaderRegexSanitizer("aeg-sas-token", "", ""),
+	newHeaderRegexSanitizer("aeg-sas-key", "", ""),
+	newHeaderRegexSanitizer("aeg-channel-name", "", ""),
+	newBodyKeySanitizer("$..adminPassword"),
+	newBodyKeySanitizer("$..administratorLoginPassword"),
+	newBodyKeySanitizer("$..accessToken"),
+	newBodyKeySanitizer("$..accessSAS"),
+	newGeneralRegexSanitizer(`(?:(sv|sig|se|srt|ss|sp)=)(?<secret>[^&\"]+)`, "secret"), // SAS tokens
+	newBodyKeySanitizer("$.value[*].key"),
+	newBodyKeySanitizer("$.key"),
+	newBodyKeySanitizer("$..userId"),
+	newBodyKeySanitizer("$..urlSource"),
+	newBodyKeySanitizer("$..uploadUrl"),
+	newBodyKeySanitizer("$..token"),
+	newBodyKeySanitizer("$..to"),
+	newBodyKeySanitizer("$..tenantId"),
+	newBodyKeySanitizer("$..targetResourceId"),
+	newBodyKeySanitizer("$..targetModelLocation"),
+	newBodyKeySanitizer("$..storageContainerWriteSas"),
+	newBodyKeySanitizer("$..storageContainerUri"),
+	newBodyKeySanitizer("$..storageContainerReadListSas"),
+	newBodyKeySanitizer("$..storageAccountPrimaryKey"),
+	newBodyKeySanitizer("$..storageAccount"),
+	newBodyKeySanitizer("$..source"),
+	newBodyKeySanitizer("$..secondaryReadonlyMasterKey"),
+	newBodyKeySanitizer("$..secondaryMasterKey"),
+	newBodyKeySanitizer("$..secondaryConnectionString"),
+	newBodyKeySanitizer("$..scriptUrlSasToken"),
+	newBodyKeySanitizer("$..scan"),
+	newBodyKeySanitizer("$..sasUri"),
+	newBodyKeySanitizer("$..resourceGroup"),
+	newBodyKeySanitizer("$..privateKey"),
+	newBodyKeySanitizer("$..principalId"),
+	newBodyKeySanitizer("$..primaryReadonlyMasterKey"),
+	newBodyKeySanitizer("$..primaryMasterKey"),
+	newBodyKeySanitizer("$..primaryConnectionString"),
+	newBodyKeySanitizer("$..password"),
+	newBodyKeySanitizer("$..outputDataUri"),
+	newBodyKeySanitizer("$..managedResourceGroupName"),
+	newBodyKeySanitizer("$..logLink"),
+	newBodyKeySanitizer("$..lastModifiedBy"),
+	newBodyKeySanitizer("$..keyVaultClientSecret"),
+	newBodyKeySanitizer("$..inputDataUri"),
+	newBodyKeySanitizer("$..id"),
+	newBodyKeySanitizer("$..httpHeader"),
+	newBodyKeySanitizer("$..guardian"),
+	newBodyKeySanitizer("$..functionKey"),
+	newBodyKeySanitizer("$..from"),
+	newBodyKeySanitizer("$..fencingClientPassword"),
+	newBodyKeySanitizer("$..encryptedCredential"),
+	newBodyKeySanitizer("$..credential"),
+	newBodyKeySanitizer("$..createdBy"),
+	newBodyKeySanitizer("$..containerUri"),
+	newBodyKeySanitizer("$..clientSecret"),
+	newBodyKeySanitizer("$..certificatePassword"),
+	newBodyKeySanitizer("$..catalog"),
+	newBodyKeySanitizer("$..azureBlobSource.containerUrl"),
+	newBodyKeySanitizer("$..authHeader"),
+	newBodyKeySanitizer("$..atlasKafkaSecondaryEndpoint"),
+	newBodyKeySanitizer("$..atlasKafkaPrimaryEndpoint"),
+	newBodyKeySanitizer("$..appkey"),
+	newBodyKeySanitizer("$..appId"),
+	newBodyKeySanitizer("$..acrToken"),
+	newBodyKeySanitizer("$..accountKey"),
+	newBodyKeySanitizer("$..AccessToken"),
+	newBodyKeySanitizer("$..WEBSITE_AUTH_ENCRYPTION_KEY"),
+	newBodyRegexSanitizer("-----BEGIN PRIVATE KEY-----\\\\n(?<key>.+\\\\n)*-----END PRIVATE KEY-----\\\\n", "key"),
+	newBodyKeySanitizer("$..adminPassword.value"),
+	newBodyKeySanitizer("$..decryptionKey"),
+	newBodyRegexSanitizer("(?<=<UserDelegationKey>).*?(?:<Value>)(.*)(?:</Value>)", ""),
+	newBodyRegexSanitizer("(?<=<UserDelegationKey>).*?(?:<SignedTid>)(.*)(?:</SignedTid>)", ""),
+	newBodyRegexSanitizer("(?<=<UserDelegationKey>).*?(?:<SignedOid>)(.*)(?:</SignedOid>)", ""),
+	newBodyRegexSanitizer("(?:Password=)(.*?)(?:;)", ""),
+	newBodyRegexSanitizer("(?:User ID=)(.+)(?:;)?", ""),
+	newBodyRegexSanitizer("(?:<PrimaryKey>)(.*)(?:</PrimaryKey>)", ""),
+	newBodyRegexSanitizer("(?:<SecondaryKey>)(.*)(?:</SecondaryKey>)", ""),
+	newBodyKeySanitizer("$..accountName"),
+	newBodyKeySanitizer("$.properties.DOCKER_REGISTRY_SERVER_PASSWORD"),
+}

sdk/internal/recording/default_sanitizers_test.go

@@ -0,0 +1,124 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package recording
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/internal/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDefaultSanitizers(t *testing.T) {
+	before := recordMode
+	defer func() { recordMode = before }()
+	recordMode = RecordingMode
+
+	t.Setenv(proxyManualStartEnv, "false")
+	o := *defaultOptions()
+	o.insecure = true
+	proxy, err := StartTestProxy("", &o)
+	require.NoError(t, err)
+	defer func() {
+		err := StopTestProxy(proxy)
+		require.NoError(t, err)
+		_ = os.Remove(filepath.Join("testdata", "recordings", t.Name()+".json"))
+	}()
+
+	client, err := NewRecordingHTTPClient(t, nil)
+	require.NoError(t, err)
+
+	srv, close := mock.NewTLSServer()
+	defer close()
+
+	// build a request and response containing all the values that should be sanitized by default
+	fail := "FAIL"
+	failSAS := strings.ReplaceAll("sv=*&sig=*&se=*&srt=*&ss=*&sp=*", "*", fail)
+	q := "?sig=" + fail
+	req, err := http.NewRequest(http.MethodGet, srv.URL()+q, nil)
+	require.NoError(t, err)
+	req.Header.Set("Content-Type", "application/json")
+	resOpts := []mock.ResponseOption{mock.WithStatusCode(http.StatusOK)}
+	body := map[string]any{}
+	for _, s := range defaultSanitizers {
+		switch s.Name {
+		case "BodyKeySanitizer":
+			k := strings.TrimLeft(s.Body.JSONPath, "$.")
+			var v any = fail
+			if before, after, found := strings.Cut(k, "."); found {
+				// path is e.g. $..foo.bar, so this value would be in a nested object
+				k = before
+				if strings.HasSuffix(k, "[*]") {
+					// path is e.g. $..foo[*].bar, so this value would be in an object array
+					k = strings.TrimSuffix(k, "[*]")
+					v = []map[string]string{{after: fail}}
+				} else {
+					v = map[string]string{after: fail}
+				}
+			}
+			body[k] = v
+		case "HeaderRegexSanitizer":
+			// if there's no group specified, we can generate a matching value because this sanitizer
+			// performs a simple replacement (this works provided the default regex sanitizers continue
+			// to follow the convention of always naming a group)
+			if s.Body.GroupForReplace == "" {
+				req.Header.Set(s.Body.Key, fail)
+				resOpts = append(resOpts, mock.WithHeader(s.Body.Key, fail))
+			}
+		default:
+			// handle regex sanitizers below because generating matching values is tricky
+		}
+	}
+	// add values matching body regex sanitizers
+	for i, v := range []string{
+		"client_secret=" + fail + "&client_assertion=" + fail,
+		strings.ReplaceAll("-----BEGIN PRIVATE KEY-----\n*\n*\n*\n-----END PRIVATE KEY-----\n", "*", fail),
+		failSAS,
+		strings.Join([]string{"AccessKey", "accesskey", "Accesskey", "AccountKey", "SharedAccessKey"}, "="+fail+";") + "=" + fail,
+		strings.ReplaceAll("<UserDelegationKey><SignedOid>*</SignedOid><SignedTid>*</SignedTid><Value>*</Value></UserDelegationKey>", "*", fail),
+		fmt.Sprintf("<PrimaryKey>%s</PrimaryKey>", failSAS),
+	} {
+		k := fmt.Sprint(i)
+		require.NotContains(t, body, k, "test bug: body already has key %q", k)
+		body[k] = v
+	}
+	// add values matching header regex sanitizers
+	for _, h := range []string{"ServiceBusDlqSupplementaryAuthorization", "ServiceBusSupplementaryAuthorization", "SupplementaryAuthorization"} {
+		req.Header.Set(h, failSAS)
+	}
+
+	// set request and response bodies
+	j, err := json.Marshal(body)
+	require.NoError(t, err)
+	req.Body = io.NopCloser(bytes.NewReader(j))
+	srv.SetResponse(append(resOpts, mock.WithBody(j))...)
+
+	err = Start(t, packagePath, nil)
+	require.NoError(t, err)
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	err = Stop(t, nil)
+	require.NoError(t, err)
+	if resp.StatusCode != http.StatusOK {
+		b, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Fatal(string(b))
+	}
+
+	b, err := os.ReadFile(fmt.Sprintf("./testdata/recordings/%s.json", t.Name()))
+	require.NoError(t, err)
+	if bytes.Contains(b, []byte(fail)) {
+		var buf bytes.Buffer
+		require.NoError(t, json.Indent(&buf, b, "", "  "))
+		t.Fatalf("%q shouldn't appear in this recording:\n%s%q shouldn't appear in the above recording", fail, buf.String(), fail)
+	}
+}