feat(azure): finish pending items for review

Author: rgmz

pkg/detectors/azure_entra/common.go

@@ -1,8 +1,10 @@
 package azure_entra
 
 import (
-	"context"
 	"fmt"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"golang.org/x/sync/singleflight"
 	"io"
 	"net/http"
 	"strings"
@@ -21,7 +23,11 @@ var (
 	// https://learn.microsoft.com/en-us/partner-center/account-settings/find-ids-and-domain-names#find-the-microsoft-azure-ad-tenant-id-and-primary-domain-name
 	// https://learn.microsoft.com/en-us/microsoft-365/admin/setup/domains-faq?view=o365-worldwide#why-do-i-have-an--onmicrosoft-com--domain
 	tenantIdPat = regexp.MustCompile(fmt.Sprintf(
-		`(?i)(?:login\.microsoftonline\.com/|sts\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,45}?)(%s)`, uuidStr,
+		//language=regexp
+		`(?i)(?:(?:login\.microsoftonline\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s))`,
+		uuidStr,
+		uuidStr,
+		uuidStr,
 	))
 	tenantOnMicrosoftPat = regexp.MustCompile(`([\w-]+\.onmicrosoft\.com)`)
 
@@ -34,7 +40,14 @@ func FindTenantIdMatches(data string) map[string]struct{} {
 	uniqueMatches := make(map[string]struct{})
 
 	for _, match := range tenantIdPat.FindAllStringSubmatch(data, -1) {
-		m := strings.ToLower(match[1])
+		var m string
+		if match[1] != "" {
+			m = strings.ToLower(match[1])
+		} else if match[2] != "" {
+			m = strings.ToLower(match[2])
+		} else if match[3] != "" {
+			m = strings.ToLower(match[3])
+		}
 		if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
 			continue
 		}
@@ -59,19 +72,56 @@ func FindClientIdMatches(data string) map[string]struct{} {
 	return uniqueMatches
 }
 
+var (
+	tenantCache = simple.NewCache[bool]()
+	tenantGroup singleflight.Group
+)
+
 // TenantExists returns whether the tenant exists according to Microsoft's well-known OpenID endpoint.
 func TenantExists(ctx context.Context, client *http.Client, tenant string) bool {
+	// Use cached value where possible.
+	if tenantExists, isCached := tenantCache.Get(tenant); isCached {
+		return tenantExists
+	}
+
+	// https://www.codingexplorations.com/blog/understanding-singleflight-in-golang-a-solution-for-eliminating-redundant-work
+	tenantExists, _, _ := tenantGroup.Do(tenant, func() (interface{}, error) {
+		result := queryTenant(ctx, client, tenant)
+		tenantCache.Set(tenant, result)
+		return result, nil
+	})
+
+	return tenantExists.(bool)
+}
+
+func queryTenant(ctx context.Context, client *http.Client, tenant string) bool {
+	logger := ctx.Logger().WithName("azure").WithValues("tenant", tenant)
+
 	tenantUrl := fmt.Sprintf("https://login.microsoftonline.com/%s/.well-known/openid-configuration", tenant)
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, tenantUrl, nil)
 	if err != nil {
 		return false
 	}
 
 	res, err := client.Do(req)
+	if err != nil {
+		logger.Error(err, "Failed to check if tenant exists")
+		return false
+	}
 	defer func() {
 		_, _ = io.Copy(io.Discard, res.Body)
 		_ = res.Body.Close()
 	}()
 
-	return res.StatusCode == http.StatusOK
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true
+	case http.StatusBadRequest:
+		logger.V(4).Info("Tenant does not exist.")
+		return false
+	default:
+		bodyBytes, _ := io.ReadAll(res.Body)
+		logger.Error(nil, "WARNING: Unexpected response when checking if tenant exists", "status_code", res.StatusCode, "body", string(bodyBytes))
+		return false
+	}
 }

pkg/detectors/azure_entra/common_test.go

@@ -35,6 +35,14 @@ func runPatTest(t *testing.T, tests map[string]testCase, matchFunc func(data str
 func Test_FindTenantIdMatches(t *testing.T) {
 	cases := map[string]testCase{
 		// Tenant ID
+		"audience": {
+			Input: `az offazure hyperv site create --location "eastus" --service-principal-identity-details \
+	application-id="cbcfc473-97da-45dd-8a00-3612d1ddf35a" \
+	audience="https://bced5192-08c4-4470-9a94-666fea59efb07/aadapp" `,
+			Expected: map[string]struct{}{
+				"bced5192-08c4-4470-9a94-666fea59efb0": {},
+			},
+		},
 		"tenant": {
 			Input: `        "cas.authn.azure-active-directory.login-url=https://login.microsoftonline.com/common/",
         "cas.authn.azure-active-directory.tenant=8e439f30-da7a-482c-bd23-e45d0a732000"`,
@@ -99,6 +107,12 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
 				"7bb339cb-e94c-4a85-884c-48ebd9bb28c3": {},
 			},
 		},
+		"login.windows.net": {
+			Input: `az offazure hyperv site create --location "eastus" --service-principal-identity-details aad-authority="https://login.windows.net/7bb339cb-e94c-4a85-884c-48ebd9bb28c3" application-id="e9f013df-2a2a-4871-b766-e79867f30348" \'`,
+			Expected: map[string]struct{}{
+				"7bb339cb-e94c-4a85-884c-48ebd9bb28c3": {},
+			},
+		},
 		"sts.windows.net": {
 			Input: `{
   "aud": "00000003-0000-0000-c000-000000000000",
@@ -108,6 +122,20 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
 				"974fde14-c3a4-481b-9b03-cfce182c3a07": {},
 			},
 		},
+		"x-anchor-mailbox": {
+			// The tenantID can be encoded in this parameter.
+			// https://github.com/AzureAD/microsoft-authentication-library-for-python/blob/95a63a7fe97d91b99979e5bf78e03f6acf40a286/msal/application.py#L185-L186
+			// https://github.com/silverhack/monkey365/blob/b3f43c4a2d014fcc3aae0a4103c8f2610fbb4980/core/utils/Get-MonkeySecCompBackendUri.ps1#L70
+			Input: `      User-Agent:
+      - python-requests/2.31.0
+      X-AnchorMailbox:
+        - Oid:2b9b0cb5-d707-42e3-9504-d9b76ac7bec5@86843c34-863b-44d3-bb14-4f14e7c0564d
+      x-client-current-telemetry:
+        - 4|84,3|`,
+			Expected: map[string]struct{}{
+				"86843c34-863b-44d3-bb14-4f14e7c0564d": {},
+			},
+		},
 
 		// Tenant onmicrosoft.com
 		"onmicrosoft tenant": {
@@ -145,6 +173,17 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
 				"12fc345b-0c67-4cde-8902-dabf2cad34b5": {},
 			},
 		},
+		"newline": {
+			Input: `        {\n   \"mode\": \"Manual\"\n  },\n  \"bootstrapProfile\": {\n   \"artifactSource\":
+        \"Direct\"\n  }\n },\n \"identity\": {\n  \"type\": \"SystemAssigned\",\n
+        \ \"principalId\":\"00000000-0000-0000-0000-000000000001\",\n  \"tenantId\":
+        \"d0a69dfd-9b9e-4833-9c33-c7903dd2e012\"\n },\n \"sku\": {\n  \"name\": \"Base\",\n
+        \ \"tier\": \"Free\"\n }\n}"
+    headers:`,
+			Expected: map[string]struct{}{
+				"d0a69dfd-9b9e-4833-9c33-c7903dd2e012": {},
+			},
+		},
 
 		// False positives
 		"tid shouldn't match clientId": {

pkg/detectors/azure_entra/serviceprincipal/sp.go

@@ -15,6 +15,7 @@ import (
 )
 
 var (
+	Description               = "Azure is a cloud service offering a wide range of services including compute, analytics, storage, and networking. Azure credentials can be used to access and manage these services."
 	ErrSecretInvalid          = errors.New("invalid client secret provided")
 	ErrSecretExpired          = errors.New("the provided secret is expired")
 	ErrTenantNotFound         = errors.New("tenant not found")
@@ -35,7 +36,6 @@ type TokenErrResponse struct {
 func VerifyCredentials(ctx context.Context, client *http.Client, tenantId string, clientId string, clientSecret string) (bool, map[string]string, error) {
 	data := url.Values{}
 	data.Set("client_id", clientId)
-	//data.Set("scope", "https://management.core.windows.net/.default")
 	data.Set("scope", "https://graph.microsoft.com/.default")
 	data.Set("client_secret", clientSecret)
 	data.Set("grant_type", "client_credentials")
@@ -91,6 +91,7 @@ func VerifyCredentials(ctx context.Context, client *http.Client, tenantId string
 		case http.StatusBadRequest, http.StatusUnauthorized:
 			// Error codes can be looked up by removing the `AADSTS` prefix.
 			// https://login.microsoftonline.com/error?code=9002313
+			// TODO: Handle AADSTS900382 (https://github.com/Azure/azure-sdk-for-js/issues/30557)
 			d := errResp.Description
 			switch {
 			case strings.HasPrefix(d, "AADSTS700016:"):

pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	"context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal"
 	"net/http"
 
 	regexp "github.com/wasilibs/go-re2"
@@ -29,7 +30,8 @@ var (
 	// TODO: Azure storage access keys and investigate other types of creds.
 	// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
 	// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
-	//clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,40}?([a-z0-9~@_\-[\]:.?]{32,34})`)
+	//clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
+	// TODO: Tighten this regex and replace it with above.
 	secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]`)
 )
 
@@ -61,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	if client == nil {
 		client = defaultClient
 	}
+	// The handling logic is identical for both versions.
 	processedResults := v2.ProcessData(ctx, clientSecrets, clientIds, tenantIds, verify, client)
 	for _, result := range processedResults {
 		results = append(results, result)
@@ -72,6 +75,10 @@ func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_Azure
 }
 
+func (s Scanner) Description() string {
+	return serviceprincipal.Description
+}
+
 // region Helper methods.
 func findSecretMatches(data string) map[string]struct{} {
 	uniqueMatches := make(map[string]struct{})