-
Notifications
You must be signed in to change notification settings - Fork 1.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Feat: Added ZohoCRM detector #3516
Merged
nabeelalam
merged 14 commits into
trufflesecurity:main
from
nabeelalam:feature/detector/zoho-crm
Nov 18, 2024
Merged
Changes from 6 commits
Commits
Show all changes
14 commits
Select commit
Hold shift + click to select a range
623cf85
added zohocrm detector and test
nabeelalam d1a1f97
separated zohocrm chunk test to integreation test file
nabeelalam 81127d4
checked out upstream/main code into branch
nabeelalam c385db1
readded zohocrm to detectors.proto file; remade protos
nabeelalam 5149f81
Merge remote-tracking branch 'upstream/main' into feature/detector/zo…
nabeelalam 1a36281
Merge remote-tracking branch 'upstream/main' into feature/detector/zo…
nabeelalam 34059d5
added zohocrm scanner in defaults.go, updated zohocrm verification en…
nabeelalam 06d23cc
Merge branch 'main' into feature/detector/zoho-crm
nabeelalam 92e4769
updated zohocrm detectors proto file
nabeelalam 24a7a68
updated zohocrm scanner description
nabeelalam 3aa981b
Merge branch 'main' into feature/detector/zoho-crm
kashifkhan0771 6a22bf0
updated zohocrm scanner in defaults.go
nabeelalam a4d1885
Merge branch 'main' into feature/detector/zoho-crm
nabeelalam edf1b10
Merge branch 'main' into feature/detector/zoho-crm
nabeelalam File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,127 @@ | ||
package zohocrm | ||
|
||
import ( | ||
"context" | ||
"encoding/json" | ||
"fmt" | ||
"io" | ||
"net/http" | ||
|
||
regexp "github.com/wasilibs/go-re2" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/common" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" | ||
) | ||
|
||
type Scanner struct { | ||
client *http.Client | ||
} | ||
|
||
type UnauthorizedResponseBody struct { | ||
Code string `json:"code"` | ||
Details map[string]interface{} `json:"details"` | ||
Message string `json:"message"` | ||
Status string `json:"status"` | ||
} | ||
|
||
// Ensure the Scanner satisfies the interface at compile time. | ||
var _ detectors.Detector = (*Scanner)(nil) | ||
|
||
var ( | ||
defaultClient = common.SaneHttpClient() | ||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives. | ||
keyPat = regexp.MustCompile(`\b(1000\.[a-f0-9]{32}\.[a-f0-9]{32})\b`) | ||
) | ||
|
||
// Keywords are used for efficiently pre-filtering chunks. | ||
// Use identifiers in the secret preferably, or the provider name. | ||
func (s Scanner) Keywords() []string { | ||
return []string{"1000."} | ||
kashifkhan0771 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
|
||
// FromData will find and optionally verify Zohocrm secrets in a given set of bytes. | ||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { | ||
dataStr := string(data) | ||
uniqueMatches := make(map[string]struct{}) | ||
|
||
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { | ||
uniqueMatches[match[1]] = struct{}{} | ||
} | ||
|
||
for match := range uniqueMatches { | ||
result := detectors.Result{ | ||
DetectorType: detectorspb.DetectorType_ZohoCRM, | ||
Raw: []byte(match), | ||
} | ||
|
||
if verify { | ||
client := s.client | ||
if client == nil { | ||
client = defaultClient | ||
} | ||
|
||
isVerified, verificationErr := verifyMatch(ctx, client, match) | ||
result.Verified = isVerified | ||
result.SetVerificationError(verificationErr, match) | ||
} | ||
|
||
results = append(results, result) | ||
} | ||
|
||
return | ||
} | ||
|
||
// Verifies the Zoho CRM API key by making an HTTP request to the Zoho CRM API. | ||
func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) { | ||
endpoint := "https://www.zohoapis.com/crm/v2/Leads" | ||
nabeelalam marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) | ||
if err != nil { | ||
return false, err | ||
} | ||
req.Header.Add("Authorization", fmt.Sprintf("Zoho-oauthtoken %s", token)) | ||
res, err := client.Do(req) | ||
if err != nil { | ||
return false, err | ||
} | ||
defer func() { | ||
_, _ = io.Copy(io.Discard, res.Body) | ||
_ = res.Body.Close() | ||
}() | ||
|
||
switch res.StatusCode { | ||
case http.StatusOK: | ||
return true, nil | ||
case http.StatusUnauthorized: | ||
bodyBytes, err := io.ReadAll(res.Body) | ||
if err != nil { | ||
return false, fmt.Errorf("failed to read response body: %v", err) | ||
} | ||
|
||
var responseBody UnauthorizedResponseBody | ||
err = json.Unmarshal(bodyBytes, &responseBody) | ||
if err != nil { | ||
return false, fmt.Errorf("failed to parse JSON response: %v", err) | ||
} | ||
|
||
switch responseBody.Code { | ||
case "OAUTH_SCOPE_MISMATCH": | ||
return true, nil | ||
case "INVALID_TOKEN": | ||
return false, nil | ||
default: | ||
return false, fmt.Errorf("unexpected error code: %s", responseBody.Code) | ||
} | ||
default: | ||
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) | ||
} | ||
} | ||
|
||
func (s Scanner) Type() detectorspb.DetectorType { | ||
return detectorspb.DetectorType_ZohoCRM | ||
} | ||
|
||
func (s Scanner) Description() string { | ||
return "Zohocrm is a blockchain development platform that provides a suite of tools and services for building and scaling decentralized applications. Zohocrm API keys can be used to access these services." | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think this is correct? |
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,161 @@ | ||
//go:build detectors | ||
// +build detectors | ||
|
||
package zohocrm | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"testing" | ||
"time" | ||
|
||
"github.com/google/go-cmp/cmp" | ||
"github.com/google/go-cmp/cmp/cmpopts" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/common" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" | ||
) | ||
|
||
func TestZohocrm_FromChunk(t *testing.T) { | ||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) | ||
defer cancel() | ||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") | ||
if err != nil { | ||
t.Fatalf("could not get test secrets from GCP: %s", err) | ||
} | ||
secret := testSecrets.MustGetField("ZOHOCRM") | ||
inactiveSecret := testSecrets.MustGetField("ZOHOCRM_INACTIVE") | ||
|
||
type args struct { | ||
ctx context.Context | ||
data []byte | ||
verify bool | ||
} | ||
tests := []struct { | ||
name string | ||
s Scanner | ||
args args | ||
want []detectors.Result | ||
wantErr bool | ||
wantVerificationErr bool | ||
}{ | ||
{ | ||
name: "found, verified", | ||
s: Scanner{}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte(fmt.Sprintf("You can find a zohocrm secret %s within", secret)), | ||
verify: true, | ||
}, | ||
want: []detectors.Result{ | ||
{ | ||
DetectorType: detectorspb.DetectorType_ZohoCRM, | ||
Verified: true, | ||
}, | ||
}, | ||
wantErr: false, | ||
wantVerificationErr: false, | ||
}, | ||
{ | ||
name: "found, unverified", | ||
s: Scanner{}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte(fmt.Sprintf("You can find a zohocrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation | ||
verify: true, | ||
}, | ||
want: []detectors.Result{ | ||
{ | ||
DetectorType: detectorspb.DetectorType_ZohoCRM, | ||
Verified: false, | ||
}, | ||
}, | ||
wantErr: false, | ||
wantVerificationErr: false, | ||
}, | ||
{ | ||
name: "not found", | ||
s: Scanner{}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte("You cannot find the secret within"), | ||
verify: true, | ||
}, | ||
want: nil, | ||
wantErr: false, | ||
wantVerificationErr: false, | ||
}, | ||
{ | ||
name: "found, would be verified if not for timeout", | ||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte(fmt.Sprintf("You can find a zohocrm secret %s within", secret)), | ||
verify: true, | ||
}, | ||
want: []detectors.Result{ | ||
{ | ||
DetectorType: detectorspb.DetectorType_ZohoCRM, | ||
Verified: false, | ||
}, | ||
}, | ||
wantErr: false, | ||
wantVerificationErr: true, | ||
}, | ||
{ | ||
name: "found, verified but unexpected api surface", | ||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte(fmt.Sprintf("You can find a zohocrm secret %s within", secret)), | ||
verify: true, | ||
}, | ||
want: []detectors.Result{ | ||
{ | ||
DetectorType: detectorspb.DetectorType_ZohoCRM, | ||
Verified: false, | ||
}, | ||
}, | ||
wantErr: false, | ||
wantVerificationErr: true, | ||
}, | ||
} | ||
for _, tt := range tests { | ||
t.Run(tt.name, func(t *testing.T) { | ||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) | ||
if (err != nil) != tt.wantErr { | ||
t.Errorf("Zohocrm.FromData() error = %v, wantErr %v", err, tt.wantErr) | ||
return | ||
} | ||
for i := range got { | ||
if len(got[i].Raw) == 0 { | ||
t.Fatalf("no raw secret present: \n %+v", got[i]) | ||
} | ||
if (got[i].VerificationError() != nil) != tt.wantVerificationErr { | ||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) | ||
} | ||
} | ||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") | ||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { | ||
t.Errorf("Zohocrm.FromData() %s diff: (-got +want)\n%s", tt.name, diff) | ||
} | ||
}) | ||
} | ||
} | ||
|
||
func BenchmarkFromData(benchmark *testing.B) { | ||
ctx := context.Background() | ||
s := Scanner{} | ||
for name, data := range detectors.MustGetBenchmarkData() { | ||
benchmark.Run(name, func(b *testing.B) { | ||
b.ResetTimer() | ||
for n := 0; n < b.N; n++ { | ||
_, err := s.FromData(ctx, false, data) | ||
if err != nil { | ||
b.Fatal(err) | ||
} | ||
} | ||
}) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
package zohocrm | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"testing" | ||
|
||
"github.com/google/go-cmp/cmp" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" | ||
) | ||
|
||
var ( | ||
validPattern = "1000.1fa6966eafbb115624baa4103269e50e.e57d155232227b4e41fa7dd2b88dd4d4" | ||
invalidPattern = "1000.24baa4103269e50e.41fa7dd2b88dd4d4" | ||
) | ||
|
||
func TestZohocrm_Pattern(t *testing.T) { | ||
d := Scanner{} | ||
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) | ||
tests := []struct { | ||
name string | ||
input string | ||
want []string | ||
}{ | ||
{ | ||
name: "typical pattern - with keyword zoho crm", | ||
input: fmt.Sprintf("zoho crm token = '%s'", validPattern), | ||
want: []string{"1000.1fa6966eafbb115624baa4103269e50e.e57d155232227b4e41fa7dd2b88dd4d4"}, | ||
}, | ||
{ | ||
name: "typical pattern - ignore duplicate", | ||
input: fmt.Sprintf("zoho crm token = '%s' | '%s'", validPattern, validPattern), | ||
want: []string{"1000.1fa6966eafbb115624baa4103269e50e.e57d155232227b4e41fa7dd2b88dd4d4"}, | ||
}, | ||
{ | ||
name: "invalid pattern", | ||
input: fmt.Sprintf("zoho crm = '%s'", invalidPattern), | ||
want: []string{}, | ||
}, | ||
} | ||
|
||
for _, test := range tests { | ||
t.Run(test.name, func(t *testing.T) { | ||
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) | ||
if len(matchedDetectors) == 0 { | ||
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) | ||
return | ||
} | ||
|
||
results, err := d.FromData(context.Background(), false, []byte(test.input)) | ||
if err != nil { | ||
t.Errorf("error = %v", err) | ||
return | ||
} | ||
|
||
if len(results) != len(test.want) { | ||
if len(results) == 0 { | ||
t.Errorf("did not receive result") | ||
} else { | ||
t.Errorf("expected %d results, only received %d", len(test.want), len(results)) | ||
} | ||
return | ||
} | ||
|
||
actual := make(map[string]struct{}, len(results)) | ||
for _, r := range results { | ||
if len(r.RawV2) > 0 { | ||
actual[string(r.RawV2)] = struct{}{} | ||
} else { | ||
actual[string(r.Raw)] = struct{}{} | ||
} | ||
} | ||
expected := make(map[string]struct{}, len(test.want)) | ||
for _, v := range test.want { | ||
expected[v] = struct{}{} | ||
} | ||
|
||
if diff := cmp.Diff(expected, actual); diff != "" { | ||
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) | ||
} | ||
}) | ||
} | ||
} |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we remove them as these 3 fields are not being used anywhere ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah I had mapped the output, in order to add extra data in the response but it felt unnecessary. I've removed this fields.