Add support for defining sources via a config file

main.go

@@ -252,6 +252,8 @@ var (
 	huggingfaceIncludeDiscussions = huggingfaceScan.Flag("include-discussions", "Include discussions in scan.").Bool()
 	huggingfaceIncludePrs         = huggingfaceScan.Flag("include-prs", "Include pull requests in scan.").Bool()
 
+	scanScan = cli.Command("scan", "Find credentials in multiple sources defined in configuration.")
+
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
 )
@@ -511,7 +513,8 @@ func run(state overseer.State) {
 	verificationCacheMetrics := verificationcache.InMemoryMetrics{}
 
 	engConf := engine.Config{
-		Concurrency: *concurrency,
+		Concurrency:       *concurrency,
+		ConfiguredSources: conf.Sources,
 		// The engine must always be configured with the list of
 		// default detectors, which can be further filtered by the
 		// user. The filters are applied by the engine and are only
@@ -698,7 +701,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		}
 	}()
 
-	var ref sources.JobProgressRef
+	var refs []sources.JobProgressRef
 	switch cmd {
 	case gitScan.FullCommand():
 		gitCfg := sources.GitConfig{
@@ -711,8 +714,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Bare:             *gitScanBare,
 			ExcludeGlobs:     *gitScanExcludeGlobs,
 		}
-		if ref, err = eng.ScanGit(ctx, gitCfg); err != nil {
+		if ref, err := eng.ScanGit(ctx, gitCfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case githubScan.FullCommand():
 		filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths)
@@ -740,8 +745,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			CommentsTimeframeDays:      *githubCommentsTimeframeDays,
 			Filter:                     filter,
 		}
-		if ref, err = eng.ScanGitHub(ctx, cfg); err != nil {
+		if ref, err := eng.ScanGitHub(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case githubExperimentalScan.FullCommand():
 		cfg := sources.GitHubExperimentalConfig{
@@ -751,8 +758,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			CollisionThreshold: *githubExperimentalCollisionThreshold,
 			DeleteCachedData:   *githubExperimentalDeleteCache,
 		}
-		if ref, err = eng.ScanGitHubExperimental(ctx, cfg); err != nil {
+		if ref, err := eng.ScanGitHubExperimental(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case gitlabScan.FullCommand():
 		filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
@@ -768,8 +777,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			ExcludeRepos: *gitlabScanExcludeRepos,
 			Filter:       filter,
 		}
-		if ref, err = eng.ScanGitLab(ctx, cfg); err != nil {
+		if ref, err := eng.ScanGitLab(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case filesystemScan.FullCommand():
 		if len(*filesystemDirectories) > 0 {
@@ -783,8 +794,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			IncludePathsFile: *filesystemScanIncludePaths,
 			ExcludePathsFile: *filesystemScanExcludePaths,
 		}
-		if ref, err = eng.ScanFileSystem(ctx, cfg); err != nil {
+		if ref, err := eng.ScanFileSystem(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case s3Scan.FullCommand():
 		cfg := sources.S3Config{
@@ -797,8 +810,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			CloudCred:     *s3ScanCloudEnv,
 			MaxObjectSize: int64(*s3ScanMaxObjectSize),
 		}
-		if ref, err = eng.ScanS3(ctx, cfg); err != nil {
+		if ref, err := eng.ScanS3(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan S3: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case syslogScan.FullCommand():
 		cfg := sources.SyslogConfig{
@@ -809,16 +824,22 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			KeyPath:     *syslogTLSKey,
 			Concurrency: *concurrency,
 		}
-		if ref, err = eng.ScanSyslog(ctx, cfg); err != nil {
+		if ref, err := eng.ScanSyslog(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case circleCiScan.FullCommand():
-		if ref, err = eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
+		if ref, err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case travisCiScan.FullCommand():
-		if ref, err = eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
+		if ref, err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case gcsScan.FullCommand():
 		cfg := sources.GCSConfig{
@@ -834,17 +855,21 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Concurrency:    *concurrency,
 			MaxObjectSize:  int64(*gcsMaxObjectSize),
 		}
-		if ref, err = eng.ScanGCS(ctx, cfg); err != nil {
+		if ref, err := eng.ScanGCS(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case dockerScan.FullCommand():
 		cfg := sources.DockerConfig{
 			BearerToken:       *dockerScanToken,
 			Images:            *dockerScanImages,
 			UseDockerKeychain: *dockerScanToken == "",
 		}
-		if ref, err = eng.ScanDocker(ctx, cfg); err != nil {
+		if ref, err := eng.ScanDocker(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case postmanScan.FullCommand():
 		// handle deprecated flag
@@ -880,8 +905,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			WorkspacePaths:      *postmanWorkspacePaths,
 			EnvironmentPaths:    *postmanEnvironmentPaths,
 		}
-		if ref, err = eng.ScanPostman(ctx, cfg); err != nil {
+		if ref, err := eng.ScanPostman(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case elasticsearchScan.FullCommand():
 		cfg := sources.ElasticsearchConfig{
@@ -896,8 +923,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			SinceTimestamp: *elasticsearchSinceTimestamp,
 			BestEffortScan: *elasticsearchBestEffortScan,
 		}
-		if ref, err = eng.ScanElasticsearch(ctx, cfg); err != nil {
+		if ref, err := eng.ScanElasticsearch(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case jenkinsScan.FullCommand():
 		cfg := engine.JenkinsConfig{
@@ -906,8 +935,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Username:              *jenkinsUsername,
 			Password:              *jenkinsPassword,
 		}
-		if ref, err = eng.ScanJenkins(ctx, cfg); err != nil {
+		if ref, err := eng.ScanJenkins(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err)
+		} else {
+			refs = append(refs, ref)
 		}
 	case huggingfaceScan.FullCommand():
 		if *huggingfaceEndpoint != "" {
@@ -939,8 +970,19 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			IncludePrs:         *huggingfaceIncludePrs,
 			Concurrency:        *concurrency,
 		}
-		if ref, err = eng.ScanHuggingface(ctx, cfg); err != nil {
+		if ref, err := eng.ScanHuggingface(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)
+		} else {
+			refs = append(refs, ref)
+		}
+	case scanScan.FullCommand():
+		if *configFilename == "" {
+			return scanMetrics, fmt.Errorf("missing required flag: --config")
+		}
+		if rs, err := eng.ScanConfig(ctx, cfg.ConfiguredSources...); err != nil {
+			return scanMetrics, fmt.Errorf("failed to scan via config: %w", err)
+		} else {
+			refs = append(refs, rs...)
 		}
 	default:
 		return scanMetrics, fmt.Errorf("invalid command: %s", cmd)
@@ -951,13 +993,19 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err)
 	}
 
-	// Print any errors reported during the scan.
-	if errs := ref.Snapshot().Errors; len(errs) > 0 {
-		errMsgs := make([]string, len(errs))
-		for i := 0; i < len(errs); i++ {
-			errMsgs[i] = errs[i].Error()
+	// Print any non-fatal errors reported during the scan.
+	for _, ref := range refs {
+		if errs := ref.Snapshot().Errors; len(errs) > 0 {
+			errMsgs := make([]string, len(errs))
+			for i := 0; i < len(errs); i++ {
+				errMsgs[i] = errs[i].Error()
+			}
+			ctx.Logger().Error(nil, "encountered errors during scan",
+				"job", ref.JobID,
+				"source_name", ref.SourceName,
+				"errors", errMsgs,
+			)
 		}
-		ctx.Logger().Error(nil, "encountered errors during scan", "errors", errMsgs)
 	}
 
 	if *printAvgDetectorTime {

pkg/config/config.go

@@ -1,16 +1,29 @@
 package config
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/custom_detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/configpb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gcs"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/jenkins"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/postman"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"
 )
 
 // Config holds user supplied configuration.
 type Config struct {
+	Sources   []sources.ConfiguredSource
 	Detectors []detectors.Detector
 }
 
@@ -25,21 +38,74 @@ func Read(filename string) (*Config, error) {
 
 // NewYAML parses the given YAML data into a Config.
 func NewYAML(input []byte) (*Config, error) {
+	var inputYAML configpb.Config
 	// Parse the raw YAML into a structure.
-	var messages custom_detectorspb.CustomDetectors
-	if err := protoyaml.UnmarshalStrict(input, &messages); err != nil {
+	if err := protoyaml.UnmarshalStrict(input, &inputYAML); err != nil {
 		return nil, err
 	}
-	// Convert the structured YAML into detectors.
-	var d []detectors.Detector
-	for _, detectorConfig := range messages.Detectors {
+
+	// Convert to detectors.
+	var detectorConfigs []detectors.Detector
+	for _, detectorConfig := range inputYAML.Detectors {
 		detector, err := custom_detectors.NewWebhookCustomRegex(detectorConfig)
 		if err != nil {
 			return nil, err
 		}
-		d = append(d, detector)
+		detectorConfigs = append(detectorConfigs, detector)
+	}
+
+	// Convert to configured sources.
+	var sourceConfigs []sources.ConfiguredSource
+	for _, pbSource := range inputYAML.Sources {
+		s, err := instantiateSourceFromType(pbSource.GetType())
+		if err != nil {
+			return nil, err
+		}
+		src := sources.NewConfiguredSource(s, pbSource)
+
+		sourceConfigs = append(sourceConfigs, src)
 	}
+
 	return &Config{
-		Detectors: d,
+		Detectors: detectorConfigs,
+		Sources:   sourceConfigs,
 	}, nil
 }
+
+// instantiateSourceFromType creates a concrete implementation of
+// sources.Source for the provided type.
+func instantiateSourceFromType(sourceType string) (sources.Source, error) {
+	var source sources.Source
+	switch sourceType {
+	case sourcespb.SourceType_SOURCE_TYPE_GIT.String():
+		source = new(git.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_GITHUB.String():
+		source = new(github.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG.String():
+		source = new(github.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT.String():
+		source = new(git.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_GITLAB.String():
+		source = new(gitlab.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_POSTMAN.String():
+		source = new(postman.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_S3.String():
+		source = new(s3.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_S3_UNAUTHED.String():
+		source = new(s3.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM.String():
+		source = new(filesystem.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_JENKINS.String():
+		source = new(jenkins.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_GCS.String():
+		source = new(gcs.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_GCS_UNAUTHED.String():
+		source = new(gcs.Source)
+	case sourcespb.SourceType_SOURCE_TYPE_DOCKER.String():
+		source = new(docker.Source)
+	default:
+		return nil, fmt.Errorf("got unexpected source type: %q", sourceType)
+	}
+
+	return source, nil
+}

pkg/engine/engine.go

@@ -99,6 +99,7 @@ type Config struct {
 	// also serves as a multiplier for other worker types (e.g., detector workers, notifier workers)
 	Concurrency int
 
+	ConfiguredSources             []sources.ConfiguredSource
 	Decoders                      []decoders.Decoder
 	Detectors                     []detectors.Detector
 	DetectorVerificationOverrides map[config.DetectorID]bool
@@ -546,6 +547,17 @@ func (r *verificationOverlapTracker) increment() {
 
 const ignoreTag = "trufflehog:ignore"
 
+// AhoCorasickCoreKeywords returns a set of keywords that the engine's
+// AhoCorasickCore is using.
+func (e *Engine) AhoCorasickCoreKeywords() map[string]struct{} {
+	// Turn AhoCorasick keywordsToDetectors into a map of keywords
+	keywords := make(map[string]struct{})
+	for key := range e.AhoCorasickCore.KeywordsToDetectors() {
+		keywords[key] = struct{}{}
+	}
+	return keywords
+}
+
 // HasFoundResults returns true if any results are found.
 func (e *Engine) HasFoundResults() bool {
 	return atomic.LoadUint32(&e.numFoundResults) > 0

pkg/engine/postman.go

@@ -39,12 +39,6 @@ func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) (sour
 		return sources.JobProgressRef{}, errors.New("no path to locally exported data or API token provided")
 	}
 
-	// Turn AhoCorasick keywordsToDetectors into a map of keywords
-	keywords := make(map[string]struct{})
-	for key := range e.AhoCorasickCore.KeywordsToDetectors() {
-		keywords[key] = struct{}{}
-	}
-
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
 	if err != nil {
@@ -56,8 +50,9 @@ func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) (sour
 	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, postman.SourceType)
 
 	postmanSource := &postman.Source{
-		DetectorKeywords: keywords,
+		DetectorKeywords: e.AhoCorasickCoreKeywords(),
 	}
+
 	if err := postmanSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
 		return sources.JobProgressRef{}, err
 	}