reverting to proxy tool

eng/pipelines/templates/steps/build-test.yml

@@ -44,7 +44,7 @@ steps:
     displayName: "Install Coverage and Junit Dependencies"
 
   - ${{ if eq(parameters.TestProxy, true) }}:
-    - template: /eng/common/testproxy/test-proxy-docker.yml
+    - template: /eng/common/testproxy/test-proxy-tool.yml
 
   - task: PowerShell@2
     displayName: 'Run Tests'
@@ -60,9 +60,8 @@ steps:
 
   - ${{ if eq(parameters.TestProxy, true) }}:
     - pwsh: |
-        # ambitious_azsdk_test_proxy is the hardcoded container name used
-        # by the test proxy startup script
-        docker logs ambitious_azsdk_test_proxy
+        # $(Build.SourcesDirectory)/test-proxy.log is the hardcoded output log location for the test-proxy-tool.yml
+        cat $(Build.SourcesDirectory)/test-proxy.log
       displayName: 'Dump Test Proxy logs'
       condition: succeededOrFailed()
 

sdk/azidentity/go.mod

@@ -2,6 +2,8 @@ module github.com/Azure/azure-sdk-for-go/sdk/azidentity
 
 go 1.16
 
+replace github.com/Azure/azure-sdk-for-go/sdk/internal => ../internal
+
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.0
 	github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.0

sdk/azidentity/go.sum

@@ -1,8 +1,5 @@
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.0 h1:8wVJL0HUP5yDFXvotdewORTw7Yu88JbreWN/mobSvsQ=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.0/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM=
-github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
-github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.0 h1:HMbyI+KfvL+XyuWekow/nWbRxsAhB6+DVzgQTIABecU=
-github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.0/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

sdk/azidentity/live_test.go

@@ -6,7 +6,6 @@ package azidentity
 import (
 	"context"
 	"fmt"
-	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -87,29 +86,29 @@ func init() {
 }
 
 func TestMain(m *testing.M) {
-	switch recording.GetRecordMode() {
-	case recording.PlaybackMode:
-		// enable BodilessMatcher because we don't record request bodies
-		// TODO: add an API for this to sdk/internal
-		req, err := http.NewRequest("POST", "http://localhost:5000/Admin/SetMatcher", http.NoBody)
+	if recording.GetRecordMode() == recording.PlaybackMode || recording.GetRecordMode() == recording.RecordingMode {
+		// Start from a fresh proxy
+		err := recording.ResetProxy(nil)
 		if err != nil {
 			panic(err)
 		}
-		req.Header["x-abstraction-identifier"] = []string{"BodilessMatcher"}
-		res, err := http.DefaultClient.Do(req)
+
+		// At the end of testing we want to reset as to not interfere with other tests.
+		defer func() {
+			err := recording.ResetProxy(nil)
+			if err != nil {
+				panic(err)
+			}
+		}()
+	}
+
+	switch recording.GetRecordMode() {
+	case recording.PlaybackMode:
+		err := recording.SetBodilessMatcher(nil, nil)
 		if err != nil {
 			panic(err)
 		}
-		if res.StatusCode != http.StatusOK {
-			log.Panicf("failed to enable BodilessMatcher: %v", res)
-		}
-		// TODO: reset matcher
 	case recording.RecordingMode:
-		// remove default sanitizers such as the OAuth response sanitizer
-		err := recording.ResetProxy(nil)
-		if err != nil {
-			panic(err)
-		}
 		// replace path variables with fake values to simplify matching (the real values aren't secret)
 		pathVars := map[string]string{
 			liveManagedIdentity.clientID:                    fakeClientID,
@@ -120,7 +119,7 @@ func TestMain(m *testing.M) {
 		}
 		for target, replacement := range pathVars {
 			if target != "" {
-				err = recording.AddURISanitizer(replacement, target, nil)
+				err := recording.AddURISanitizer(replacement, target, nil)
 				if err != nil {
 					panic(err)
 				}
@@ -139,7 +138,7 @@ func TestMain(m *testing.M) {
 		// remove token request bodies (which are form encoded) because they contain
 		// secrets, are irrelevant in matching, and are formed by MSAL anyway
 		// (note: Cloud Shell would need an exemption from this, and that would be okay--its requests contain no secrets)
-		err = recording.AddBodyRegexSanitizer("{}", `^\S+=.*`, nil)
+		err := recording.AddBodyRegexSanitizer("{}", `^\S+=.*`, nil)
 		if err != nil {
 			panic(err)
 		}
@@ -150,15 +149,10 @@ func TestMain(m *testing.M) {
 				panic(err)
 			}
 		}
-		defer func() {
-			err := recording.ResetProxy(nil)
-			if err != nil {
-				panic(err)
-			}
-			// TODO: reset matcher
-		}()
 	}
-	os.Exit(m.Run())
+	val := m.Run()
+
+	os.Exit(val)
 }
 
 func initRecording(t *testing.T) (policy.ClientOptions, func()) {

sdk/data/aztables/go.mod

@@ -2,6 +2,8 @@ module github.com/Azure/azure-sdk-for-go/sdk/data/aztables
 
 go 1.16
 
+replace github.com/Azure/azure-sdk-for-go/sdk/internal => ../../internal
+
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.13.0

sdk/data/aztables/go.sum

@@ -2,9 +2,6 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.0 h1:8wVJL0HUP5yDFXvotdewORTw
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.0/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.13.0 h1:bLRntPH25SkY1uZ/YZW+dmxNky9r1fAHvDFrzluo+4Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.13.0/go.mod h1:TmXReXZ9yPp5D5TBRMTAtyz+UyOl15Py4hL5E5p6igQ=
-github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
-github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.0 h1:HMbyI+KfvL+XyuWekow/nWbRxsAhB6+DVzgQTIABecU=
-github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.0/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

sdk/data/aztables/proxy_test.go

@@ -21,7 +21,15 @@ import (
 
 func TestMain(m *testing.M) {
 	// 1. Set up session level sanitizers
-	if recording.GetRecordMode() == "record" {
+	switch recording.GetRecordMode() {
+	case recording.PlaybackMode:
+		err := recording.SetDefaultMatcher(nil, &recording.SetDefaultMatcherOptions{
+			ExcludedHeaders: []string{":path", ":auth", ":method", ":scheme"},
+		})
+		if err != nil {
+			panic(err)
+		}
+	case recording.RecordingMode:
 		for _, val := range []string{"TABLES_COSMOS_ACCOUNT_NAME", "TABLES_STORAGE_ACCOUNT_NAME"} {
 			account, ok := os.LookupEnv(val)
 			if !ok {
@@ -34,8 +42,8 @@ func TestMain(m *testing.M) {
 				panic(err)
 			}
 		}
-	}
 
+	}
 	// Run tests
 	exitVal := m.Run()
 

sdk/internal/CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 0.9.1 (Unreleased)
 
 ### Features Added
+* Adds a `CustomDefaultMatcher` that adds headers `:path`, `:authority`, `:method`, and `:scheme` to the 
 
 ### Breaking Changes
 

sdk/internal/recording/matchers.go

@@ -7,8 +7,11 @@
 package recording
 
 import (
-	"fmt"
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
 	"net/http"
+	"strings"
 	"testing"
 )
 
@@ -19,24 +22,90 @@ type MatcherOptions struct {
 // SetBodilessMatcher adjusts the "match" operation to exclude the body when matching a request to a recording's entries.
 // Pass in `nil` for `t` if you want the bodiless matcher to apply everywhere
 func SetBodilessMatcher(t *testing.T, options *MatcherOptions) error {
+	f := false
+	return SetDefaultMatcher(t, &SetDefaultMatcherOptions{
+		CompareBodies: &f,
+	})
+}
+
+type SetDefaultMatcherOptions struct {
+	CompareBodies       *bool
+	ExcludedHeaders     []string
+	IgnoredHeaders      []string
+	IgnoreQueryOrdering *bool
+}
+
+func (s *SetDefaultMatcherOptions) fillOptions() {
+	f := false
+	t := true
+	if s == nil {
+		s = &SetDefaultMatcherOptions{
+			CompareBodies:       &t,
+			IgnoreQueryOrdering: &f,
+		}
+		return
+	}
+
+	if s.CompareBodies == nil {
+		s.CompareBodies = &t
+	}
+	if s.IgnoreQueryOrdering == nil {
+		s.IgnoreQueryOrdering = &f
+	}
+}
+
+func addDefaults(added []string) []string {
+	if added == nil {
+		return nil
+	}
+	needToAdd := []string{":path", ":authority", ":method", ":scheme"}
+	for _, a := range added {
+		for idx, n := range needToAdd {
+			if a == n {
+				needToAdd = append(needToAdd[:idx], needToAdd[idx+1:]...)
+			}
+		}
+	}
+	return append(added, needToAdd...)
+}
+
+// SetDefaultMatcher adjusts the "match" operation to exclude the body when matching a request to a recording's entries.
+// Pass in `nil` for `t` if you want the bodiless matcher to apply everywhere
+func SetDefaultMatcher(t *testing.T, options *SetDefaultMatcherOptions) error {
 	if recordMode != PlaybackMode {
 		return nil
 	}
+	options.fillOptions()
 	req, err := http.NewRequest("POST", "http://localhost:5000/Admin/SetMatcher", http.NoBody)
 	if err != nil {
 		panic(err)
 	}
-	req.Header["x-abstraction-identifier"] = []string{"BodilessMatcher"}
+	req.Header["x-abstraction-identifier"] = []string{"CustomDefaultMatcher"}
 	if t != nil {
 		req.Header["x-recording-id"] = []string{GetRecordingId(t)}
 	}
 
-	res, err := http.DefaultClient.Do(req)
+	if !(*options.CompareBodies) {
+		options.ExcludedHeaders = append(options.ExcludedHeaders, "Content-Length")
+	}
+
+	marshalled, err := json.MarshalIndent(struct {
+		CompareBodies       *bool  `json:"compareBodies,omitempty"`
+		ExcludedHeaders     string `json:"excludedHeaders,omitempty"`
+		IncludedHeaders     string `json:"includedHeaders,omitempty"`
+		IgnoreQueryOrdering *bool  `json:"ignoreQueryOrdering,omitempty"`
+	}{
+		CompareBodies:       options.CompareBodies,
+		ExcludedHeaders:     strings.Join(addDefaults(options.ExcludedHeaders), ","),
+		IncludedHeaders:     strings.Join(options.IgnoredHeaders, ","),
+		IgnoreQueryOrdering: options.IgnoreQueryOrdering,
+	}, "", "")
 	if err != nil {
 		return err
 	}
-	if res.StatusCode != http.StatusOK {
-		return fmt.Errorf("failed to enable BodilessMatcher: %v", res)
-	}
-	return nil
+
+	req.Body = ioutil.NopCloser(bytes.NewReader(marshalled))
+	req.ContentLength = int64(len(marshalled))
+
+	return handleProxyResponse(client.Do(req))
 }

sdk/internal/recording/matchers_test.go

@@ -113,3 +113,63 @@ func TestSetBodilessMatcherNilTest(t *testing.T) {
 	err = ResetProxy(nil)
 	require.NoError(t, err)
 }
+
+func TestSetDefaultMatcher(t *testing.T) {
+	temp := recordMode
+	recordMode = RecordingMode
+	defer func() { recordMode = temp }()
+
+	err := Start(t, packagePath, nil)
+	require.NoError(t, err)
+
+	req, err := http.NewRequest("POST", "https://localhost:5001", nil)
+	require.NoError(t, err)
+
+	req.Header.Set(UpstreamURIHeader, "https://bing.com")
+	req.Header.Set(ModeHeader, GetRecordMode())
+	req.Header.Set(IDHeader, GetRecordingId(t))
+
+	client, err := GetHTTPClient(t)
+	require.NoError(t, err)
+
+	_, err = client.Do(req)
+	require.NoError(t, err)
+
+	err = Stop(t, nil)
+	require.NoError(t, err)
+
+	// Run a second request to with different body to verify it works
+	recordMode = PlaybackMode
+
+	err = Start(t, packagePath, nil)
+	require.NoError(t, err)
+
+	err = SetDefaultMatcher(nil, &SetDefaultMatcherOptions{ExcludedHeaders: []string{"ExampleHeader"}})
+	require.NoError(t, err)
+
+	req, err = http.NewRequest("POST", "https://localhost:5001", nil)
+	require.NoError(t, err)
+
+	req.Header.Set(UpstreamURIHeader, "https://bing.com")
+	req.Header.Set(ModeHeader, GetRecordMode())
+	req.Header.Set(IDHeader, GetRecordingId(t))
+	req.Header.Set("ExampleHeader", "blah-blah-blah")
+
+	err = handleProxyResponse(client.Do(req))
+	require.NoError(t, err)
+
+	err = Stop(t, nil)
+	require.NoError(t, err)
+
+	err = ResetProxy(nil)
+	require.NoError(t, err)
+}
+
+func TestAddDefaults(t *testing.T) {
+	require.Equal(t, 4, len(addDefaults([]string{})))
+	require.Equal(t, 4, len(addDefaults([]string{":path"})))
+	require.Equal(t, 4, len(addDefaults([]string{":path", ":authority"})))
+	require.Equal(t, 4, len(addDefaults([]string{":path", ":authority", ":method"})))
+	require.Equal(t, 4, len(addDefaults([]string{":path", ":authority", ":method", ":scheme"})))
+	require.Equal(t, 5, len(addDefaults([]string{":path", ":authority", ":method", ":scheme", "extra"})))
+}

sdk/internal/recording/recording.go

@@ -476,6 +476,22 @@ func init() {
 	if ok := certPool.AppendCertsFromPEM(cert); !ok {
 		log.Println("no certs appended, using system certs only")
 	}
+
+	// Set a Default matcher that ignores :path, :scheme, :authority, and :method headers
+	err = SetDefaultMatcher(
+		nil,
+		&SetDefaultMatcherOptions{ExcludedHeaders: []string{
+			":authority",
+			":method",
+			":path",
+			":scheme",
+		}},
+	)
+	if err != nil {
+		log.Println("could not set the default matcher")
+	} else {
+		log.Println("default matcher was set ")
+	}
 }
 
 var recordMode string