diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index a0a2ce22419f..a34cd1d43bba 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -64,8 +64,6 @@ steps: - ${{ if eq(parameters.TestProxy, true) }}: - template: /eng/common/testproxy/test-proxy-tool.yml - parameters: - targetVersion: '1.0.0-dev.20230427.1' - task: PowerShell@2 displayName: 'Run Tests' diff --git a/sdk/internal/CHANGELOG.md b/sdk/internal/CHANGELOG.md index a31046c07fb3..fdcb4317c787 100644 --- a/sdk/internal/CHANGELOG.md +++ b/sdk/internal/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features Added * Options types for `SetBodilessMatcher` and `SetDefaultMatcher` now embed `RecordingOptions` +* Added a collection of default sanitizers for test recordings ### Breaking Changes diff --git a/sdk/internal/recording/default_sanitizers.go b/sdk/internal/recording/default_sanitizers.go new file mode 100644 index 000000000000..53eb0c7ae70f --- /dev/null +++ b/sdk/internal/recording/default_sanitizers.go @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// this file contains a set of default sanitizers applied to all recordings + +package recording + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// sanitizer represents a single sanitizer configured via the test proxy's /Admin/AddSanitizers endpoint +type sanitizer struct { + // Name is the name of a sanitizer type e.g. "BodyKeySanitizer" + Name string `json:"Name,omitempty"` + Body sanitizerBody `json:"Body,omitempty"` +} + +type sanitizerBody struct { + // GroupForReplace is the name of the regex group to replace + GroupForReplace string `json:"groupForReplace,omitempty"` + // JSONPath is the JSON path to the value to replace + JSONPath string `json:"jsonPath,omitempty"` + // Key is the name of a header to sanitize + Key string `json:"key,omitempty"` + // Regex is the regular expression to match a value to sanitize + Regex string `json:"regex,omitempty"` + // Value is the string that replaces the matched value. The sanitizers in + // this file accept the test proxy's default Value, "Sanitized". + Value string `json:"value,omitempty"` +} + +func newBodyKeySanitizer(jsonPath string) sanitizer { + return sanitizer{ + Name: "BodyKeySanitizer", + Body: sanitizerBody{ + JSONPath: jsonPath, + }, + } +} + +func newBodyRegexSanitizer(regex, groupForReplace string) sanitizer { + return sanitizer{ + Name: "BodyRegexSanitizer", + Body: sanitizerBody{ + GroupForReplace: groupForReplace, + Regex: regex, + }, + } +} + +func newGeneralRegexSanitizer(regex, groupForReplace string) sanitizer { + return sanitizer{ + Name: "GeneralRegexSanitizer", + Body: sanitizerBody{ + GroupForReplace: groupForReplace, + Regex: regex, + }, + } +} + +func newHeaderRegexSanitizer(key, regex, groupForReplace string) sanitizer { + return sanitizer{ + Name: "HeaderRegexSanitizer", + Body: sanitizerBody{ + GroupForReplace: groupForReplace, + Key: key, + Regex: regex, + }, + } +} + +// addSanitizers adds an arbitrary number of sanitizers with a single request. It +// isn't exported because SDK modules don't add enough sanitizers to benefit from it. +func addSanitizers(s []sanitizer, options *RecordingOptions) error { + if options == nil { + options = defaultOptions() + } + url := fmt.Sprintf("%s/Admin/AddSanitizers", options.baseURL()) + req, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return err + } + handleTestLevelSanitizer(req, options) + b, err := json.Marshal(s) + if err != nil { + return err + } + req.Body = io.NopCloser(bytes.NewReader(b)) + req.ContentLength = int64(len(b)) + req.Header.Set("Content-Type", "application/json") + return handleProxyResponse(client.Do(req)) +} + +var defaultSanitizers = []sanitizer{ + newGeneralRegexSanitizer(`("|;)Secret=(?[^;]+)`, "secret"), + newBodyKeySanitizer("$..refresh_token"), + newHeaderRegexSanitizer("api-key", "", ""), + newBodyKeySanitizer("$..access_token"), + newBodyKeySanitizer("$..connectionString"), + newBodyKeySanitizer("$..applicationSecret"), + newBodyKeySanitizer("$..apiKey"), + newBodyRegexSanitizer(`client_secret=(?[^&"]+)`, "secret"), + newBodyRegexSanitizer(`client_assertion=(?[^&"]+)`, "secret"), + newHeaderRegexSanitizer("x-ms-rename-source", "", ""), + newHeaderRegexSanitizer("x-ms-file-rename-source-authorization", "", ""), + newHeaderRegexSanitizer("x-ms-file-rename-source", "", ""), + newHeaderRegexSanitizer("x-ms-encryption-key-sha256", "", ""), + newHeaderRegexSanitizer("x-ms-encryption-key", "", ""), + newHeaderRegexSanitizer("x-ms-copy-source-authorization", "", ""), + newHeaderRegexSanitizer("x-ms-copy-source", "", ""), + newBodyRegexSanitizer("token=(?[^&]+)($|&)", "token"), + newHeaderRegexSanitizer("subscription-key", "", ""), + newBodyKeySanitizer("$..sshPassword"), + newBodyKeySanitizer("$..secondaryKey"), + newBodyKeySanitizer("$..runAsPassword"), + newBodyKeySanitizer("$..primaryKey"), + newHeaderRegexSanitizer("Location", "", ""), + newGeneralRegexSanitizer(`("|;)[Aa]ccess[Kk]ey=(?[^;]+)`, "secret"), + newGeneralRegexSanitizer(`("|;)[Aa]ccount[Kk]ey=(?[^;]+)`, "secret"), + newBodyKeySanitizer("$..aliasSecondaryConnectionString"), + newGeneralRegexSanitizer(`("|;)[Ss]hared[Aa]ccess[Kk]ey=(?[^;\"]+)`, "secret"), + newHeaderRegexSanitizer("aeg-sas-token", "", ""), + newHeaderRegexSanitizer("aeg-sas-key", "", ""), + newHeaderRegexSanitizer("aeg-channel-name", "", ""), + newBodyKeySanitizer("$..adminPassword"), + newBodyKeySanitizer("$..administratorLoginPassword"), + newBodyKeySanitizer("$..accessToken"), + newBodyKeySanitizer("$..accessSAS"), + newGeneralRegexSanitizer(`(?:(sv|sig|se|srt|ss|sp)=)(?[^&\"]+)`, "secret"), // SAS tokens + newBodyKeySanitizer("$.value[*].key"), + newBodyKeySanitizer("$.key"), + newBodyKeySanitizer("$..userId"), + newBodyKeySanitizer("$..urlSource"), + newBodyKeySanitizer("$..uploadUrl"), + newBodyKeySanitizer("$..token"), + newBodyKeySanitizer("$..to"), + newBodyKeySanitizer("$..tenantId"), + newBodyKeySanitizer("$..targetResourceId"), + newBodyKeySanitizer("$..targetModelLocation"), + newBodyKeySanitizer("$..storageContainerWriteSas"), + newBodyKeySanitizer("$..storageContainerUri"), + newBodyKeySanitizer("$..storageContainerReadListSas"), + newBodyKeySanitizer("$..storageAccountPrimaryKey"), + newBodyKeySanitizer("$..storageAccount"), + newBodyKeySanitizer("$..source"), + newBodyKeySanitizer("$..secondaryReadonlyMasterKey"), + newBodyKeySanitizer("$..secondaryMasterKey"), + newBodyKeySanitizer("$..secondaryConnectionString"), + newBodyKeySanitizer("$..scriptUrlSasToken"), + newBodyKeySanitizer("$..scan"), + newBodyKeySanitizer("$..sasUri"), + newBodyKeySanitizer("$..resourceGroup"), + newBodyKeySanitizer("$..privateKey"), + newBodyKeySanitizer("$..principalId"), + newBodyKeySanitizer("$..primaryReadonlyMasterKey"), + newBodyKeySanitizer("$..primaryMasterKey"), + newBodyKeySanitizer("$..primaryConnectionString"), + newBodyKeySanitizer("$..password"), + newBodyKeySanitizer("$..outputDataUri"), + newBodyKeySanitizer("$..managedResourceGroupName"), + newBodyKeySanitizer("$..logLink"), + newBodyKeySanitizer("$..lastModifiedBy"), + newBodyKeySanitizer("$..keyVaultClientSecret"), + newBodyKeySanitizer("$..inputDataUri"), + newBodyKeySanitizer("$..id"), + newBodyKeySanitizer("$..httpHeader"), + newBodyKeySanitizer("$..guardian"), + newBodyKeySanitizer("$..functionKey"), + newBodyKeySanitizer("$..from"), + newBodyKeySanitizer("$..fencingClientPassword"), + newBodyKeySanitizer("$..encryptedCredential"), + newBodyKeySanitizer("$..credential"), + newBodyKeySanitizer("$..createdBy"), + newBodyKeySanitizer("$..containerUri"), + newBodyKeySanitizer("$..clientSecret"), + newBodyKeySanitizer("$..certificatePassword"), + newBodyKeySanitizer("$..catalog"), + newBodyKeySanitizer("$..azureBlobSource.containerUrl"), + newBodyKeySanitizer("$..authHeader"), + newBodyKeySanitizer("$..atlasKafkaSecondaryEndpoint"), + newBodyKeySanitizer("$..atlasKafkaPrimaryEndpoint"), + newBodyKeySanitizer("$..appkey"), + newBodyKeySanitizer("$..appId"), + newBodyKeySanitizer("$..acrToken"), + newBodyKeySanitizer("$..accountKey"), + newBodyKeySanitizer("$..AccessToken"), + newBodyKeySanitizer("$..WEBSITE_AUTH_ENCRYPTION_KEY"), + newBodyRegexSanitizer("-----BEGIN PRIVATE KEY-----\\\\n(?.+\\\\n)*-----END PRIVATE KEY-----\\\\n", "key"), + newBodyKeySanitizer("$..adminPassword.value"), + newBodyKeySanitizer("$..decryptionKey"), + newBodyRegexSanitizer("(?<=).*?(?:)(.*)(?:)", ""), + newBodyRegexSanitizer("(?<=).*?(?:)(.*)(?:)", ""), + newBodyRegexSanitizer("(?<=).*?(?:)(.*)(?:)", ""), + newBodyRegexSanitizer("(?:Password=)(.*?)(?:;)", ""), + newBodyRegexSanitizer("(?:User ID=)(.+)(?:;)?", ""), + newBodyRegexSanitizer("(?:)(.*)(?:)", ""), + newBodyRegexSanitizer("(?:)(.*)(?:)", ""), + newBodyKeySanitizer("$..accountName"), + newBodyKeySanitizer("$.properties.DOCKER_REGISTRY_SERVER_PASSWORD"), +} diff --git a/sdk/internal/recording/default_sanitizers_test.go b/sdk/internal/recording/default_sanitizers_test.go new file mode 100644 index 000000000000..26a18b1a4275 --- /dev/null +++ b/sdk/internal/recording/default_sanitizers_test.go @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package recording + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/internal/mock" + "github.com/stretchr/testify/require" +) + +func TestDefaultSanitizers(t *testing.T) { + before := recordMode + defer func() { recordMode = before }() + recordMode = RecordingMode + + t.Setenv(proxyManualStartEnv, "false") + o := *defaultOptions() + o.insecure = true + proxy, err := StartTestProxy("", &o) + require.NoError(t, err) + defer func() { + err := StopTestProxy(proxy) + require.NoError(t, err) + _ = os.Remove(filepath.Join("testdata", "recordings", t.Name()+".json")) + }() + + client, err := NewRecordingHTTPClient(t, nil) + require.NoError(t, err) + + srv, close := mock.NewTLSServer() + defer close() + + // build a request and response containing all the values that should be sanitized by default + fail := "FAIL" + failSAS := strings.ReplaceAll("sv=*&sig=*&se=*&srt=*&ss=*&sp=*", "*", fail) + q := "?sig=" + fail + req, err := http.NewRequest(http.MethodGet, srv.URL()+q, nil) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + resOpts := []mock.ResponseOption{mock.WithStatusCode(http.StatusOK)} + body := map[string]any{} + for _, s := range defaultSanitizers { + switch s.Name { + case "BodyKeySanitizer": + k := strings.TrimLeft(s.Body.JSONPath, "$.") + var v any = fail + if before, after, found := strings.Cut(k, "."); found { + // path is e.g. $..foo.bar, so this value would be in a nested object + k = before + if strings.HasSuffix(k, "[*]") { + // path is e.g. $..foo[*].bar, so this value would be in an object array + k = strings.TrimSuffix(k, "[*]") + v = []map[string]string{{after: fail}} + } else { + v = map[string]string{after: fail} + } + } + body[k] = v + case "HeaderRegexSanitizer": + // if there's no group specified, we can generate a matching value because this sanitizer + // performs a simple replacement (this works provided the default regex sanitizers continue + // to follow the convention of always naming a group) + if s.Body.GroupForReplace == "" { + req.Header.Set(s.Body.Key, fail) + resOpts = append(resOpts, mock.WithHeader(s.Body.Key, fail)) + } + default: + // handle regex sanitizers below because generating matching values is tricky + } + } + // add values matching body regex sanitizers + for i, v := range []string{ + "client_secret=" + fail + "&client_assertion=" + fail, + strings.ReplaceAll("-----BEGIN PRIVATE KEY-----\n*\n*\n*\n-----END PRIVATE KEY-----\n", "*", fail), + failSAS, + strings.Join([]string{"AccessKey", "accesskey", "Accesskey", "AccountKey", "SharedAccessKey"}, "="+fail+";") + "=" + fail, + strings.ReplaceAll("***", "*", fail), + fmt.Sprintf("%s", failSAS), + } { + k := fmt.Sprint(i) + require.NotContains(t, body, k, "test bug: body already has key %q", k) + body[k] = v + } + // add values matching header regex sanitizers + for _, h := range []string{"ServiceBusDlqSupplementaryAuthorization", "ServiceBusSupplementaryAuthorization", "SupplementaryAuthorization"} { + req.Header.Set(h, failSAS) + } + + // set request and response bodies + j, err := json.Marshal(body) + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(j)) + srv.SetResponse(append(resOpts, mock.WithBody(j))...) + + err = Start(t, packagePath, nil) + require.NoError(t, err) + resp, err := client.Do(req) + require.NoError(t, err) + err = Stop(t, nil) + require.NoError(t, err) + if resp.StatusCode != http.StatusOK { + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + t.Fatal(string(b)) + } + + b, err := os.ReadFile(fmt.Sprintf("./testdata/recordings/%s.json", t.Name())) + require.NoError(t, err) + if bytes.Contains(b, []byte(fail)) { + var buf bytes.Buffer + require.NoError(t, json.Indent(&buf, b, "", " ")) + t.Fatalf("%q shouldn't appear in this recording:\n%s%q shouldn't appear in the above recording", fail, buf.String(), fail) + } +} diff --git a/sdk/internal/recording/recording.go b/sdk/internal/recording/recording.go index 22dc78f6a5de..56d3a53c0202 100644 --- a/sdk/internal/recording/recording.go +++ b/sdk/internal/recording/recording.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -84,7 +85,12 @@ const ( Secret_Base64String VariableType = "secret_base64String" ) +// defaultSanitizersSet tracks whether default sanitizers have been added (this would be neater with Go 1.19's atomic.Bool) +var defaultSanitizersSet int32 + // NewRecording initializes a new Recording instance +// +// Deprecated: call [Start] instead func NewRecording(c TestContext, mode RecordMode) (*Recording, error) { // create recorder based on the test name, recordMode, variables, and sanitizers recPath, varPath := getFilePaths(c.Name()) @@ -528,6 +534,10 @@ type RecordingOptions struct { GroupForReplace string Variables map[string]interface{} TestInstance *testing.T + + // insecure allows this package's tests to configure the proxy to skip upstream TLS + // verification so they can use a mock upstream server having a self-signed cert + insecure bool } func defaultOptions() *RecordingOptions { @@ -674,9 +684,6 @@ func requestStart(url string, testId string, assetConfigLocation string) (*http. } func Start(t *testing.T, pathToRecordings string, options *RecordingOptions) error { - if options == nil { - options = defaultOptions() - } if recordMode == LiveMode { return nil } @@ -687,7 +694,16 @@ func Start(t *testing.T, pathToRecordings string, options *RecordingOptions) err return nil } } - + if recordMode == RecordingMode && atomic.CompareAndSwapInt32(&defaultSanitizersSet, 0, 1) { + err := addSanitizers(defaultSanitizers, options) + if err != nil { + atomic.StoreInt32(&defaultSanitizersSet, 0) + return err + } + } + if options == nil { + options = defaultOptions() + } testId := getTestId(pathToRecordings, t) absAssetLocation, relAssetLocation, err := getAssetsConfigLocation(pathToRecordings) diff --git a/sdk/internal/recording/sanitizer.go b/sdk/internal/recording/sanitizer.go index b5c5b6cdcf40..2ddaf7119f6e 100644 --- a/sdk/internal/recording/sanitizer.go +++ b/sdk/internal/recording/sanitizer.go @@ -106,7 +106,7 @@ func handleProxyResponse(resp *http.Response, err error) error { if err != nil { return err } - return fmt.Errorf("there was an error communicating with the test proxy: %s", body) + return fmt.Errorf("%s responded %s: %s", resp.Request.URL, resp.Status, body) } // handleTestLevelSanitizer sets the "x-recording-id" header if options.TestInstance is not nil diff --git a/sdk/internal/recording/server.go b/sdk/internal/recording/server.go index 23dea92baa16..a17c8bffe369 100644 --- a/sdk/internal/recording/server.go +++ b/sdk/internal/recording/server.go @@ -409,11 +409,14 @@ func StartTestProxy(pathToRecordings string, options *RecordingOptions) (*TestPr if options == nil { options = defaultOptions() } - log.Printf("Running test proxy command: %s start --storage-location %s -- --urls=%s\n", - proxyPath, gitRoot, options.baseURL()) + insecure := "" + if options.insecure { + insecure = "--insecure" + } + args := []string{"start", "--storage-location", gitRoot, insecure, "--", "--urls=" + options.baseURL()} + log.Printf("Running test proxy command: %s %s", proxyPath, strings.Join(args, " ")) log.Printf("Test proxy log location: %s\n", proxyLog.Name()) - cmd := exec.Command( - proxyPath, "start", "--storage-location", gitRoot, "--", "--urls="+options.baseURL()) + cmd := exec.Command(proxyPath, args...) cmd.Stdout = proxyLog cmd.Stderr = proxyLog