speedup skip-dirs processing by caching

Author: jirfag

pkg/lint/runner.go

@@ -75,7 +75,7 @@ func NewRunner(astCache *astcache.Cache, cfg *config.Config, log logutils.Log, g
 			skipDirsProcessor, // must be after path prettifier
 
 			processors.NewAutogeneratedExclude(astCache),
-			processors.NewIdentifierMarker(), // must be befor exclude
+			processors.NewIdentifierMarker(), // must be before exclude because users see already marked output and configure excluding by it
 			processors.NewExclude(excludeTotalPattern),
 			processors.NewExcludeRules(excludeRules, lineCache, log.Child("exclude_rules")),
 			processors.NewNolint(astCache, log.Child("nolint"), dbManager),
@@ -92,6 +92,8 @@ func NewRunner(astCache *astcache.Cache, cfg *config.Config, log logutils.Log, g
 	}, nil
 }
 
+func someUnusedFunc() {}
+
 type lintRes struct {
 	linter *linter.Config
 	err    error
@@ -217,13 +219,19 @@ func (r *Runner) runWorkers(ctx context.Context, lintCtx *linter.Context, linter
 	return lintResultsCh
 }
 
+type processorStat struct {
+	inCount  int
+	outCount int
+}
+
 func (r Runner) processLintResults(inCh <-chan lintRes) <-chan lintRes {
 	outCh := make(chan lintRes, 64)
 
 	go func() {
 		sw := timeutils.NewStopwatch("processing", r.Log)
 
 		var issuesBefore, issuesAfter int
+		statPerProcessor := map[string]processorStat{}
 		defer close(outCh)
 
 		for res := range inCh {
@@ -234,7 +242,7 @@ func (r Runner) processLintResults(inCh <-chan lintRes) <-chan lintRes {
 
 			if len(res.issues) != 0 {
 				issuesBefore += len(res.issues)
-				res.issues = r.processIssues(res.issues, sw)
+				res.issues = r.processIssues(res.issues, sw, statPerProcessor)
 				issuesAfter += len(res.issues)
 				outCh <- res
 			}
@@ -252,12 +260,23 @@ func (r Runner) processLintResults(inCh <-chan lintRes) <-chan lintRes {
 		if issuesBefore != issuesAfter {
 			r.Log.Infof("Issues before processing: %d, after processing: %d", issuesBefore, issuesAfter)
 		}
+		r.printPerProcessorStat(statPerProcessor)
 		sw.PrintStages()
 	}()
 
 	return outCh
 }
 
+func (r Runner) printPerProcessorStat(stat map[string]processorStat) {
+	parts := make([]string, 0, len(stat))
+	for name, ps := range stat {
+		if ps.inCount != 0 {
+			parts = append(parts, fmt.Sprintf("%s: %d/%d", name, ps.outCount, ps.inCount))
+		}
+	}
+	r.Log.Infof("Processors filtering stat (out/in): %s", strings.Join(parts, ", "))
+}
+
 func collectIssues(resCh <-chan lintRes) <-chan result.Issue {
 	retIssues := make(chan result.Issue, 1024)
 	go func() {
@@ -294,7 +313,7 @@ func (r Runner) Run(ctx context.Context, linters []*linter.Config, lintCtx *lint
 	return collectIssues(processedLintResultsCh)
 }
 
-func (r *Runner) processIssues(issues []result.Issue, sw *timeutils.Stopwatch) []result.Issue {
+func (r *Runner) processIssues(issues []result.Issue, sw *timeutils.Stopwatch, statPerProcessor map[string]processorStat) []result.Issue {
 	for _, p := range r.Processors {
 		var newIssues []result.Issue
 		var err error
@@ -306,6 +325,10 @@ func (r *Runner) processIssues(issues []result.Issue, sw *timeutils.Stopwatch) [
 		if err != nil {
 			r.Log.Warnf("Can't process result by %s processor: %s", p.Name(), err)
 		} else {
+			stat := statPerProcessor[p.Name()]
+			stat.inCount += len(issues)
+			stat.outCount += len(newIssues)
+			statPerProcessor[p.Name()] = stat
 			issues = newIssues
 		}
 

pkg/result/processors/skip_dirs.go

@@ -11,11 +11,17 @@ import (
 	"github.com/golangci/golangci-lint/pkg/result"
 )
 
+type skipStat struct {
+	pattern string
+	count   int
+}
+
 type SkipDirs struct {
-	patterns    []*regexp.Regexp
-	log         logutils.Log
-	skippedDirs map[string]string // dir to the last regexp mapping
-	absArgsDirs []string
+	patterns         []*regexp.Regexp
+	log              logutils.Log
+	skippedDirs      map[string]*skipStat
+	absArgsDirs      []string
+	skippedDirsCache map[string]bool
 }
 
 var _ Processor = SkipFiles{}
@@ -50,14 +56,15 @@ func NewSkipDirs(patterns []string, log logutils.Log, runArgs []string) (*SkipDi
 	}
 
 	return &SkipDirs{
-		patterns:    patternsRe,
-		log:         log,
-		skippedDirs: map[string]string{},
-		absArgsDirs: absArgsDirs,
+		patterns:         patternsRe,
+		log:              log,
+		skippedDirs:      map[string]*skipStat{},
+		absArgsDirs:      absArgsDirs,
+		skippedDirsCache: map[string]bool{},
 	}, nil
 }
 
-func (p SkipDirs) Name() string {
+func (p *SkipDirs) Name() string {
 	return "skip_dirs"
 }
 
@@ -78,12 +85,26 @@ func (p *SkipDirs) shouldPassIssue(i *result.Issue) bool {
 	}
 
 	issueRelDir := filepath.Dir(i.FilePath())
+
+	if toPass, ok := p.skippedDirsCache[issueRelDir]; ok {
+		if !toPass {
+			p.skippedDirs[issueRelDir].count++
+		}
+		return toPass
+	}
+
 	issueAbsDir, err := filepath.Abs(issueRelDir)
 	if err != nil {
 		p.log.Warnf("Can't abs-ify path %q: %s", issueRelDir, err)
 		return true
 	}
 
+	toPass := p.shouldPassIssueDirs(issueRelDir, issueAbsDir)
+	p.skippedDirsCache[issueRelDir] = toPass
+	return toPass
+}
+
+func (p *SkipDirs) shouldPassIssueDirs(issueRelDir, issueAbsDir string) bool {
 	for _, absArgDir := range p.absArgsDirs {
 		if absArgDir == issueAbsDir {
 			// we must not skip issues if they are from explicitly set dirs
@@ -101,16 +122,21 @@ func (p *SkipDirs) shouldPassIssue(i *result.Issue) bool {
 	for _, pattern := range p.patterns {
 		if pattern.MatchString(issueRelDir) {
 			ps := pattern.String()
-			p.skippedDirs[issueRelDir] = ps
+			if p.skippedDirs[issueRelDir] == nil {
+				p.skippedDirs[issueRelDir] = &skipStat{
+					pattern: ps,
+				}
+			}
+			p.skippedDirs[issueRelDir].count++
 			return false
 		}
 	}
 
 	return true
 }
 
-func (p SkipDirs) Finish() {
-	for dir, pattern := range p.skippedDirs {
-		p.log.Infof("Skipped dir %s by pattern %s", dir, pattern)
+func (p *SkipDirs) Finish() {
+	for dir, stat := range p.skippedDirs {
+		p.log.Infof("Skipped %d issues from dir %s by pattern %s", stat.count, dir, stat.pattern)
 	}
 }