Refactored Gitlab Detectors

pkg/detectors/gitlab/v1/gitlab.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"maps"
 	"net/http"
 	"strings"
 
@@ -38,12 +39,28 @@ var (
 	BlockedUserMessage = "403 Forbidden - Your account has been blocked"
 )
 
+func (s Scanner) getClient() *http.Client {
+	if s.client != nil {
+		return s.client
+	}
+
+	return defaultClient
+}
+
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
 	return []string{"gitlab"}
 }
 
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Gitlab
+}
+
+func (s Scanner) Description() string {
+	return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab API tokens can be used to access and modify repository data and other resources."
+}
+
 // FromData will find and optionally verify Gitlab secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
@@ -72,14 +89,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			isVerified, extraData, analysisInfo, verificationErr := s.verifyGitlab(ctx, resMatch)
-			s1.Verified = isVerified
-			for key, value := range extraData {
-				s1.ExtraData[key] = value
+			for _, endpoint := range s.Endpoints() {
+				isVerified, extraData, verificationErr := VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)
+				s1.Verified = isVerified
+				maps.Copy(s1.ExtraData, extraData)
+
+				s1.SetVerificationError(verificationErr)
+
+				// for verified keys set the analysis info
+				if s1.Verified {
+					s1.AnalysisInfo = map[string]string{
+						"key":  resMatch,
+						"host": endpoint,
+					}
+				}
 			}
-
-			s1.SetVerificationError(verificationErr, resMatch)
-			s1.AnalysisInfo = analysisInfo
 		}
 
 		results = append(results, s1)
@@ -88,74 +112,51 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
-func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, map[string]string, error) {
+func VerifyGitlab(ctx context.Context, client *http.Client, baseEndpoint, resMatch string) (bool, map[string]string, error) {
 	// there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry
 	// they all grant access to different parts of the API. I couldn't find an endpoint that every
 	// one of these scopes has access to, so we just check an example endpoint for each scope. If any
 	// of them contain data, we know we have a valid key, but if they all fail, we don't
 
-	client := s.client
-	if client == nil {
-		client = defaultClient
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseEndpoint+"/api/v4/user", http.NoBody)
+	if err != nil {
+		return false, nil, err
 	}
-	for _, baseURL := range s.Endpoints() {
-		// test `read_user` scope
-		req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
-		if err != nil {
-			continue
-		}
-
-		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-		res, err := client.Do(req)
-		if err != nil {
-			return false, nil, nil, err
-		}
-
-		defer res.Body.Close()
 
-		bodyBytes, err := io.ReadAll(res.Body)
-		if err != nil {
-			return false, nil, nil, err
-		}
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+	res, err := client.Do(req)
+	if err != nil {
+		return false, nil, err
+	}
 
-		analysisInfo := map[string]string{
-			"key":  resMatch,
-			"host": baseURL,
-		}
+	defer res.Body.Close()
 
-		// 200 means good key and has `read_user` scope
-		// 403 means good key but not the right scope
-		// 401 is bad key
-		switch res.StatusCode {
-		case http.StatusOK:
-			return json.Valid(bodyBytes), nil, analysisInfo, nil
-		case http.StatusForbidden:
-			// check if the user account is blocked or not
-			stringBody := string(bodyBytes)
-			if strings.Contains(stringBody, BlockedUserMessage) {
-				return true, map[string]string{
-					"blocked": "True",
-				}, analysisInfo, nil
-			}
+	bodyBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		return false, nil, err
+	}
 
-			// Good key but not the right scope
-			return true, nil, analysisInfo, nil
-		case http.StatusUnauthorized:
-			// Nothing to do; zero values are the ones we want
-			return false, nil, nil, nil
-		default:
-			return false, nil, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	// 200 means good key and has `read_user` scope
+	// 403 means good key but not the right scope
+	// 401 is bad key
+	switch res.StatusCode {
+	case http.StatusOK:
+		return json.Valid(bodyBytes), nil, nil
+	case http.StatusForbidden:
+		// check if the user account is blocked or not
+		stringBody := string(bodyBytes)
+		if strings.Contains(stringBody, BlockedUserMessage) {
+			return true, map[string]string{
+				"blocked": "True",
+			}, nil
 		}
 
+		// Good key but not the right scope
+		return true, nil, nil
+	case http.StatusUnauthorized:
+		// Nothing to do; zero values are the ones we want
+		return false, nil, nil
+	default:
+		return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 	}
-
-	return false, nil, nil, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Gitlab
-}
-
-func (s Scanner) Description() string {
-	return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab API tokens can be used to access and modify repository data and other resources."
 }

pkg/detectors/gitlab/v1/gitlab_integration_test.go

@@ -189,7 +189,7 @@ func TestGitlab_FromChunk(t *testing.T) {
 				}
 				got[i].AnalysisInfo = nil
 			}
-			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
 			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
 				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}

pkg/detectors/gitlab/v2/gitlab_integration_test.go

@@ -101,7 +101,7 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {
 				}
 				got[i].AnalysisInfo = nil
 			}
-			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
 			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
 				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
@@ -280,7 +280,7 @@ func TestGitlabV2_FromChunk(t *testing.T) {
 				}
 				got[i].AnalysisInfo = nil
 			}
-			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
 			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
 				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}

pkg/detectors/gitlab/v2/gitlab_v2.go

@@ -3,7 +3,7 @@ package gitlab
 import (
 	"context"
 	"fmt"
-	"io"
+	"maps"
 	"net/http"
 	"strings"
 
@@ -33,10 +33,26 @@ var (
 	keyPat        = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
 )
 
+func (s Scanner) getClient() *http.Client {
+	if s.client != nil {
+		return s.client
+	}
+
+	return defaultClient
+}
+
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string { return []string{"glpat-"} }
 
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Gitlab
+}
+
+func (s Scanner) Description() string {
+	return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab Personal Access Tokens (PATs) can be used to authenticate and access GitLab resources."
+}
+
 // FromData will find and optionally verify Gitlab secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
@@ -56,87 +72,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			isVerified, extraData, analysisInfo, verificationErr := s.verifyGitlab(ctx, resMatch)
-			s1.Verified = isVerified
-			for key, value := range extraData {
-				s1.ExtraData[key] = value
+			for _, endpoint := range s.Endpoints() {
+				isVerified, extraData, verificationErr := v1.VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)
+				s1.Verified = isVerified
+				maps.Copy(s1.ExtraData, extraData)
+
+				s1.SetVerificationError(verificationErr)
+
+				// for verified keys set the analysis info
+				if s1.Verified {
+					s1.AnalysisInfo = map[string]string{
+						"key":  resMatch,
+						"host": endpoint,
+					}
+				}
 			}
-
-			s1.SetVerificationError(verificationErr, resMatch)
-			s1.AnalysisInfo = analysisInfo
 		}
 
 		results = append(results, s1)
 	}
 
 	return results, nil
 }
-
-func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, map[string]string, error) {
-	// there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry
-	// they all grant access to different parts of the API. I couldn't find an endpoint that every
-	// one of these scopes has access to, so we just check an example endpoint for each scope. If any
-	// of them contain data, we know we have a valid key, but if they all fail, we don't
-
-	client := s.client
-	if client == nil {
-		client = defaultClient
-	}
-	for _, baseURL := range s.Endpoints() {
-		// test `read_user` scope
-		req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
-		if err != nil {
-			continue
-		}
-		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-		res, err := client.Do(req)
-		if err != nil {
-			return false, nil, nil, err
-		}
-		defer res.Body.Close()
-
-		bodyBytes, err := io.ReadAll(res.Body)
-		if err != nil {
-			return false, nil, nil, err
-		}
-
-		analysisInfo := map[string]string{
-			"key":  resMatch,
-			"host": baseURL,
-		}
-
-		// 200 means good key and has `read_user` scope
-		// 403 means good key but not the right scope
-		// 401 is bad key
-		switch res.StatusCode {
-		case http.StatusOK:
-			return true, nil, analysisInfo, nil
-		case http.StatusForbidden:
-			// check if the user account is blocked or not
-			stringBody := string(bodyBytes)
-			if strings.Contains(stringBody, v1.BlockedUserMessage) {
-				return true, map[string]string{
-					"blocked": "True",
-				}, analysisInfo, nil
-			}
-
-			// Good key but not the right scope
-			return true, nil, analysisInfo, nil
-		case http.StatusUnauthorized:
-			// Nothing to do; zero values are the ones we want
-			return false, nil, nil, nil
-		default:
-			return false, nil, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
-		}
-
-	}
-	return false, nil, nil, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Gitlab
-}
-
-func (s Scanner) Description() string {
-	return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab Personal Access Tokens (PATs) can be used to authenticate and access GitLab resources."
-}