Implements option in linter test framework to run the tests with an m…

…llint.Runner, implements custom linter running its custom rules on the runner

linters/custom/linter.go

@@ -27,7 +27,7 @@ type CustomLinter struct {
 }
 
 func (l *CustomLinter) Name() string {
-	return "Custom"
+	return "Custom Rules"
 }
 
 func (l *CustomLinter) Configure(conf *config.Config) error {
@@ -54,44 +54,68 @@ func (l *CustomLinter) SetRunner(runner mllint.Runner) {
 func (l *CustomLinter) LintProject(project api.Project) (api.Report, error) {
 	report := api.NewReport()
 
-	var multiErr *multierror.Error
+	// create linters from each of the rules and schedule each of them for execution on the mllint.Runner
+	tasks := []*mllint.RunnerTask{}
 	for customRule, rule := range l.customRules {
-		var err error
-		report.Scores[*rule], report.Details[*rule], err = l.runCustomRule(project, customRule)
-		if err != nil {
-			multiErr = multierror.Append(multiErr, err)
-			continue
-		}
+		customLinter := customRuleLinter{customRule, *rule}
+		task := l.runner.RunLinter(rule.Slug, &customLinter, project)
+		tasks = append(tasks, task)
 	}
 
+	// collect all the results
+	var multiErr *multierror.Error
+	mllint.ForEachTask(l.runner.CollectTasks(tasks...), func(task *mllint.RunnerTask, result mllint.LinterResult) {
+		report = api.MergeReports(report, result.Report)
+		multiErr = multierror.Append(multiErr, result.Err)
+	})
+
 	return report, multiErr.ErrorOrNil()
 }
 
-func (l *CustomLinter) runCustomRule(project api.Project, rule config.CustomRule) (float64, string, error) {
-	// TODO: run these rules in their own processes on a runner
+//---------------------------------------------------------------------------------------
+
+// describes the expected structure of what the execution of a custom rule should result in
+type customRuleResult struct {
+	Score   float64 `json:"score" yaml:"score"`
+	Details string  `json:"details" yaml:"details"`
+}
+
+//---------------------------------------------------------------------------------------
+
+type customRuleLinter struct {
+	customRule config.CustomRule
+	rule       api.Rule
+}
 
-	cmdparts, err := shlex.Split(rule.Run)
+func (l *customRuleLinter) Name() string {
+	return "Custom Rule - " + l.rule.Name
+}
+
+// otherwise unused, but it's here to ensure customRuleLinter implements api.Linter
+func (l *customRuleLinter) Rules() []*api.Rule { return []*api.Rule{&l.rule} }
+
+// runs the custom rule definition's `run` command in the project's root directory and parses the result as YAML ()
+func (l *customRuleLinter) LintProject(project api.Project) (api.Report, error) {
+	report := api.NewReport()
+
+	cmdparts, err := shlex.Split(l.customRule.Run)
 	if err != nil {
-		return 0, "", fmt.Errorf("custom rule `%s` has invalid run command `%s`: %w", rule.Slug, rule.Run, err)
+		return report, fmt.Errorf("custom rule `%s` has invalid run command `%s`: %w", l.customRule.Slug, l.customRule.Run, err)
 	}
 
 	output, err := exec.CommandCombinedOutput(project.Dir, cmdparts[0], cmdparts[1:]...)
 	if err != nil {
-		return 0, "", fmt.Errorf("custom rule `%s` was run, but exited with an error: %w.%s", rule.Slug, err, formatOutput(output))
+		return report, fmt.Errorf("custom rule `%s` was run, but exited with an error: %w.%s", l.customRule.Slug, err, formatOutput(output))
 	}
 
 	var result customRuleResult
 	if err := yaml.Unmarshal(output, &result); err != nil {
-		return 0, "", fmt.Errorf("custom rule `%s` executed successfully, but the output was not a valid JSON / YAML object: %w.%s", rule.Slug, err, formatOutput(output))
+		return report, fmt.Errorf("custom rule `%s` executed successfully, but the output was not a valid YAML or JSON object: %w.%s", l.customRule.Slug, err, formatOutput(output))
 	}
 
-	return result.Score, result.Details, nil
-}
-
-// describes the expected structure of what the execution of a custom rule should result in
-type customRuleResult struct {
-	Score   float64 `json:"score" yaml:"score"`
-	Details string  `json:"details" yaml:"details"`
+	report.Scores[l.rule] = result.Score
+	report.Details[l.rule] = result.Details
+	return report, nil
 }
 
 // formats the output of an executed command such that it can be appended to an error.

linters/custom/linter_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/bvobart/mllint/api"
+	"github.com/bvobart/mllint/commands/mllint"
 	"github.com/bvobart/mllint/config"
 	"github.com/bvobart/mllint/linters/custom"
 	"github.com/bvobart/mllint/linters/testutils"
@@ -13,10 +14,19 @@ import (
 
 func TestCustomLinter(t *testing.T) {
 	linter := custom.NewLinter()
-	require.Equal(t, "Custom", linter.Name())
+	require.Equal(t, "Custom Rules", linter.Name())
 	require.Equal(t, []*api.Rule(nil), linter.Rules())
 
 	suite := testutils.NewLinterTestSuite(linter, []testutils.LinterTest{
+		{
+			Name:    "NoCustomRules",
+			Dir:     ".",
+			Options: testutils.NewOptions().WithConfig(config.Default()),
+			Expect: func(t *testing.T, report api.Report, err error) {
+				require.NoError(t, err)
+				require.Equal(t, api.NewReport(), report)
+			},
+		},
 		{
 			Name: "SimpleEcho",
 			Dir:  ".",
@@ -85,11 +95,16 @@ func TestCustomLinter(t *testing.T) {
 				require.Contains(t, err.Error(), "custom rule `custom/error-rule-3` was run, but exited with an error: exit status 1")
 				require.Contains(t, err.Error(), "custom rule `custom/error-rule-4` was run, but exited with an error: exit status 1")
 				require.Contains(t, err.Error(), "```\ndate: invalid option -- 'e'\nTry 'date --help' for more information.\n```")
-				require.Contains(t, err.Error(), "custom rule `custom/error-rule-5` executed successfully, but the output was not a valid JSON / YAML object: yaml: did not find expected key. Output: `score 100, details: \"\" }`")
+				require.Contains(t, err.Error(), "custom rule `custom/error-rule-5` executed successfully, but the output was not a valid YAML or JSON object: yaml: did not find expected key. Output: `score 100, details: \"\" }`")
 			},
 		},
 	})
-	suite.DefaultOptions().WithConfig(config.Default())
+
+	runner := mllint.NewMLLintRunner(nil)
+	runner.Start()
+	defer runner.Close()
+
+	suite.DefaultOptions().WithConfig(config.Default()).WithRunner(runner)
 	suite.RunAll(t)
 }
 

linters/testutils/options.go

@@ -2,12 +2,14 @@ package testutils
 
 import (
 	"github.com/bvobart/mllint/api"
+	"github.com/bvobart/mllint/commands/mllint"
 	"github.com/bvobart/mllint/config"
 	"github.com/bvobart/mllint/utils"
 )
 
 type LinterTestOptions struct {
 	conf              *config.Config
+	runner            mllint.Runner
 	detectPythonFiles bool
 	detectDepManagers bool
 	detectCQLinters   bool
@@ -54,3 +56,8 @@ func (opts *LinterTestOptions) WithConfig(conf *config.Config) *LinterTestOption
 	opts.conf = conf
 	return opts
 }
+
+func (opts *LinterTestOptions) WithRunner(runner mllint.Runner) *LinterTestOptions {
+	opts.runner = runner
+	return opts
+}

linters/testutils/suite.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/bvobart/mllint/api"
+	"github.com/bvobart/mllint/commands/mllint"
 	"github.com/bvobart/mllint/setools/cqlinters"
 	"github.com/bvobart/mllint/setools/depmanagers"
 	"github.com/bvobart/mllint/utils"
@@ -64,6 +65,7 @@ func (suite *LinterTestSuite) applyOptions(t *testing.T, testOptions *LinterTest
 	suite.applyDepManagerOptions(t, testOptions, project)
 	suite.applyCQLinterOptions(t, testOptions, project)
 	suite.applyConfigOption(t, testOptions)
+	suite.applyRunnerOption(testOptions)
 }
 
 //---------------------------------------------------------------------------------------
@@ -132,8 +134,24 @@ func (suite *LinterTestSuite) applyConfigOption(t *testing.T, testOptions *Linte
 	}
 }
 
+func (suite *LinterTestSuite) applyRunnerOption(testOptions *LinterTestOptions) {
+	if rlinter, ok := suite.linter.(mllint.LinterWithRunner); ok {
+		if testOptions != nil && testOptions.runner != nil {
+			rlinter.SetRunner(testOptions.runner)
+			return
+		}
+
+		if suite.defaultOpts != nil && suite.defaultOpts.conf != nil {
+			rlinter.SetRunner(suite.defaultOpts.runner)
+			return
+		}
+	}
+}
+
 func (suite *LinterTestSuite) canBeParallelised(test LinterTest) bool {
 	_, isConfigurable := suite.linter.(api.ConfigurableLinter)
+	_, wantsRunner := suite.linter.(mllint.LinterWithRunner)
 	needsConfig := (suite.defaultOpts != nil && suite.defaultOpts.conf != nil) || (test.Options != nil && test.Options.conf != nil)
-	return !(isConfigurable && needsConfig)
+	needsRunner := (suite.defaultOpts != nil && suite.defaultOpts.runner != nil) || (test.Options != nil && test.Options.runner != nil)
+	return !(isConfigurable && needsConfig) && !(wantsRunner && needsRunner)
 }