Merge pull request #25 from bvobart/custom-linter

Implements linter and configuration for running custom user-defined linting rules

ReadMe.md

@@ -132,11 +132,23 @@ mllint describe version-control/data/dvc
 mllint describe version-control/data
 ```
 
+### Custom linting rules
+
+It is also possible to define your own custom linting rules by implementing a script or program that `mllint` will run while performing its analysis.
+These custom rules need to be defined in `mllint`'s configuration. For more information on how to do this, see `mllint describe custom`.
+
 ---
 
 ## Configuration
 
-`mllint` can be configured either using a `.mllint.yml` file or through the project's `pyproject.toml`. This allows you to selectively disable specific linting rules or categories by means of their slug, as well as configure specific settings for various linting rules. See below for examples of such configuration files.
+`mllint` can be configured either using a `.mllint.yml` file or through the project's `pyproject.toml`. This allows you to:
+- selectively disable specific linting rules or categories using their slug
+- define custom linting rules
+- configure specific settings for various linting rules.
+
+See the code snippets and commands provided below for examples of such configuration files.
+
+#### Commands
 
 To print `mllint`'s current configuration in YAML format, run (optionally providing the path to the project's folder):
 ```sh
@@ -173,7 +185,6 @@ If no `.mllint.yml` is found, `mllint` searches the project's `pyproject.toml` f
 An example `pyproject.toml` configuration of `mllint` is as follows. Note that it is identical to the YAML example above.
 
 ```toml
-[tool.mllint]
 [tool.mllint.rules]
 disabled = ["version-control/code/git", "dependency-management/single"]
 ```

api/api_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/bvobart/mllint/api"
 	"github.com/bvobart/mllint/categories"
+	"github.com/bvobart/mllint/config"
 )
 
 func TestCategoryString(t *testing.T) {
@@ -67,3 +68,20 @@ func TestMergeReports(t *testing.T) {
 
 	require.Equal(t, expectedReport, finalReport)
 }
+
+func TestNewCustomRule(t *testing.T) {
+	cr := config.CustomRule{
+		Name:    "Custom Test Rule",
+		Slug:    "custom/test-rule",
+		Details: "Tests whether parsing a custom rule from a YAML config works",
+		Weight:  420,
+		Run:     "python ./scripts/mllint-test-rule.py",
+	}
+	rule := api.NewCustomRule(cr)
+
+	require.Equal(t, cr.Name, rule.Name)
+	require.Equal(t, cr.Slug, rule.Slug)
+	require.Equal(t, cr.Details, rule.Details)
+	require.Equal(t, cr.Weight, rule.Weight)
+	require.False(t, rule.Disabled)
+}

api/rule.go

@@ -1,5 +1,7 @@
 package api
 
+import "github.com/bvobart/mllint/config"
+
 // Rule is a struct for defining what a rule looks like that `mllint` will check.
 type Rule struct {
 	// Slug should be a lowercased, dashed reference code, e.g. 'git-no-big-files'
@@ -27,3 +29,14 @@ func (r *Rule) Disable() {
 func (r *Rule) Enable() {
 	r.Disabled = false
 }
+
+// NewCustomRule creates a new rule based on a custom rule definition as can be configured in `mllint`'s config
+func NewCustomRule(cr config.CustomRule) Rule {
+	return Rule{
+		Slug:     cr.Slug,
+		Name:     cr.Name,
+		Details:  cr.Details,
+		Weight:   cr.Weight,
+		Disabled: false,
+	}
+}

categories/categories.go

@@ -171,6 +171,75 @@ var Deployment = api.Category{
 It is not yet implemented, but may contain rules about Dockerfiles and configurability, among others.`,
 }
 
+var Custom = api.Category{
+	Name: "Custom Rules",
+	Slug: "custom",
+	Description: "This category enables you to write your own custom evaluation rules for `mllint`. " + `
+Custom rules can be useful for enforcing team, company or organisational practices,
+as well as implementing checks and analyses for how your proprietary / closed-source tools are being used.
+Custom rules may also be useful for creating 'plugins' to ` + "`mllint`" + `, that implement checks on tools
+that ` + "`mllint`" + ` does not yet have built-in rules for.
+
+` + "`mllint`" + ` will pick up these custom rules from your configuration and automatically run their checks during its analysis.
+It is also possible to use the ` + "`mllint describe`" + ` command with custom rules. Similarly, ` + "`mllint list all`" + ` also lists all custom linting rules.
+
+---
+
+To create such a custom rule, write a script or program that checks whether your project adheres to a certain practice and prints a simple YAML or JSON object
+containing the score for this rule, possibly along with some detail text. Then, add the rule's name, slug, details and run command
+to your project's ` + "`mllint`" + ` config.
+
+The following snippet of a YAML ` + "`mllint`" + ` configuration is an example of how a custom rule can be configured.
+See the table below for more details about each of the custom rule definition's properties.
+` + "```yaml" + `
+rules:
+  custom:
+    - name: Project contains a LICENSE file
+      slug: custom/is-licensed
+      details: This rule checks whether the project contains a LICENSE or LICENSE.md file at the project's root.
+      weight: 1
+      run: bash ./scripts/mllint/check-license.sh
+` + "```" + `
+
+The equivalent configuration in TOML syntax (for ` + "`pyproject.toml`" + ` files) is as follows. Multiple of these snippets can be repeated for defining more rules.
+
+` + "```toml" + `
+[[tool.mllint.rules.custom]]
+name = "Project contains a LICENSE file"
+slug = "custom/is-licensed"
+details = "This rule checks whether the project contains a LICENSE or LICENSE.md file at the project's root."
+weight = 1.0
+run = "bash ./scripts/mllint/check-license.sh"
+` + "```" + `
+
+Property | Type | Description
+---------|------|-------------
+` + "`name`" + ` | string | A short and concise sentence on what this rule expects of a project / what the rule enforces on the project. Feel free to take inspiration from the names given to ` + "`mllint`'s" + ` built-in rules.
+` + "`slug`" + ` | string | A unique and URL-friendly identifier for each rule. Should only consist of lowercased letters with dashes for spaces, optionally using slashes for categorisation. For custom rule definitions, the recommended convention is for their slugs to always start with ` + "`custom/`" + `
+` + "`details`" + ` | string | A longer, descriptive, Markdown-formatted text that explains the rule in more detail. This text should explain... _1)_ what exactly the rule checks; _2)_ why the rule checks what it checks, i.e., why is this practice important?; and 3) how should a user fix violations of this rule?
+` + "`weight`" + ` | float | The weight of this rule compared to other rules in the same category. This is used for calculating the category score as a weighted average of the scores of all rules. Zero weight means the rule's results will be shown in the report, but won't count for the category score. Note that YAML accepts any number for this property, e.g. ` + "`4`" + `, but TOML is more strict with typing and requires you to specify a number with a decimal point, e.g. ` + "`4.0`" + `
+` + "`run`" + ` | string | The command to run for evaluating this rule. This command will be run in the project's root directory. Note that the command will be run using Golang's ` + "[`os/exec`](https://pkg.go.dev/os/exec)" + ` package, which _"intentionally does not invoke the system shell and does not expand any glob patterns or handle other expansions, pipelines, or redirections typically done by shells."_ To run shell commands, invoke the shell directly using e.g. ` + "`bash -c 'your command && here'`" + ` or using the example above to execute shell scripts.
+
+The command specified with ` + "`run`" + ` is expected to print a simple YAML (or JSON) object with the following structure: 
+
+Property | Type | Description
+---------|------|-------------
+` + "`score`" + ` | float | The score given to the rule. Must be a number between 0 and 100, i.e., the score is a percentage indicating the degree to which the project adheres to the implemented rule.
+` + "`details`" + ` | string | A Markdown-formatted piece of text that provides details about the given score and what the user can do to fix a violation of this rule. Where applicable, you may also use this to congratulate the user on successful implementation of this rule.
+
+For an example implementation, consider the rule defined in the example configuration above. The script below is a possible implementation of the ` + "`./scripts/mllint/check-license.sh`" + ` script that is referred to by the example.
+
+` + "```bash" + `
+#!/bin/bash
+if [[ -f LICENSE ]] || [[ -f LICENSE.md ]]; then
+  echo 'score: 100'
+else
+  echo 'score: 0'
+  echo 'details: "Your project is missing a LICENSE. Please be sure to include our [company license file](https://link.to/company/license-file/) in your project."'
+fi
+` + "```",
+}
+
 var All = []api.Category{
 	VersionControl,
 	FileStructure,
@@ -180,6 +249,7 @@ var All = []api.Category{
 	Testing,
 	ContinuousIntegration,
 	Deployment,
+	Custom,
 }
 
 var BySlug = makeSlugMap()

commands/describe.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/bvobart/mllint/api"
 	"github.com/bvobart/mllint/categories"
+	"github.com/bvobart/mllint/config"
 	"github.com/bvobart/mllint/linters"
 	"github.com/bvobart/mllint/utils/markdown"
 	"github.com/bvobart/mllint/utils/markdowngen"
@@ -38,6 +39,17 @@ func describe(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	conf, conftype, err := config.ParseFromDir(".")
+	if err == nil && len(conf.Rules.Custom) > 0 {
+		shush(func() {
+			color.Green("Including %d custom rules from the %s file in the current directory\n\n", len(conf.Rules.Custom), conftype.String())
+		})
+
+		if err := linters.ConfigureAll(conf); err != nil {
+			color.HiYellow("Warning! The mllint configuration file %s contains an error: %s\n\n", conftype.String(), err.Error())
+		}
+	}
+
 	output := strings.Builder{}
 	for i, slug := range args {
 		if i > 0 {
@@ -49,7 +61,8 @@ func describe(cmd *cobra.Command, args []string) error {
 		} else if rules := linters.FindRules(slug); len(rules) > 0 {
 			output.WriteString(describeRules(rules))
 		} else {
-			output.WriteString(color.RedString("No rule or category found that matched: %s\n", color.Set(color.Reset).Sprint(slug)))
+			output.WriteString(color.RedString("No rule or category found that matched: %s\n", color.New(color.FgYellow, color.Italic).Sprint(slug)))
+			continue
 		}
 
 		if outputToFile() || outputToStdout() {
@@ -65,11 +78,7 @@ func describe(cmd *cobra.Command, args []string) error {
 		return writeToOutputFile(output.String())
 	}
 
-	if outputToStdout() {
-		fmt.Println(output.String())
-		return nil
-	}
-
+	fmt.Println(output.String())
 	return nil
 }
 

commands/list.go

@@ -7,6 +7,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/bvobart/mllint/api"
+	"github.com/bvobart/mllint/config"
 	"github.com/bvobart/mllint/linters"
 	"github.com/bvobart/mllint/utils/markdowngen"
 )
@@ -60,6 +61,18 @@ func listAll(_ *cobra.Command, args []string) error {
 	if err := checkOutputFlag(); err != nil {
 		return err
 	}
+
+	conf, conftype, err := config.ParseFromDir(".")
+	if err == nil && len(conf.Rules.Custom) > 0 {
+		shush(func() {
+			color.Green("Including %d custom rules from the %s file in the current directory\n\n", len(conf.Rules.Custom), conftype.String())
+		})
+
+		if err := linters.ConfigureAll(conf); err != nil {
+			color.HiYellow("Warning! The mllint configuration file %s contains an error: %s\n\n", conftype.String(), err.Error())
+		}
+	}
+
 	return listLinters(linters.ByCategory)
 }
 
@@ -80,10 +93,10 @@ func listEnabled(_ *cobra.Command, args []string) error {
 	}
 	shush(func() { fmt.Print("---\n\n") })
 
-	linters.DisableAll(conf.Rules.Disabled)
 	if err := linters.ConfigureAll(conf); err != nil {
 		return err
 	}
+	linters.DisableAll(conf.Rules.Disabled)
 
 	if err := listLinters(linters.ByCategory); err != nil {
 		return err

commands/run.go

@@ -85,14 +85,15 @@ func (rc *runCommand) RunLint(cmd *cobra.Command, args []string) error {
 	rc.ProjectR.Config = *rc.Config
 	shush(func() { fmt.Print("---\n\n") })
 
-	// disable any rules from config
-	rulesDisabled := linters.DisableAll(rc.Config.Rules.Disabled)
-
 	// configure all linters with config
 	if err = linters.ConfigureAll(rc.Config); err != nil {
 		return err
 	}
 
+	// disable any rules from config.
+	// This is done after configuring each linter, such that any rules arising from the configuration (e.g. custom rules) can also be disabled.
+	rulesDisabled := linters.DisableAll(rc.Config.Rules.Disabled)
+
 	// run pre-analysis checks
 	if err = rc.runPreAnalysisChecks(); err != nil {
 		return fmt.Errorf("failed to run pre-analysis checks: %w", err)

config/config.go

@@ -21,24 +21,42 @@ type Config struct {
 	Testing     TestingConfig     `yaml:"testing" toml:"testing"`
 }
 
+//---------------------------------------------------------------------------------------
+
 // RuleConfig contains info about which rules are enabled / disabled.
 type RuleConfig struct {
-	Disabled []string `yaml:"disabled" toml:"disabled"`
+	Disabled []string     `yaml:"disabled" toml:"disabled"`
+	Custom   []CustomRule `yaml:"custom" toml:"custom"`
+}
+
+// CustomRule contains the configuration for custom rules
+type CustomRule struct {
+	Name    string  `yaml:"name" toml:"name"`
+	Slug    string  `yaml:"slug" toml:"slug"`
+	Details string  `yaml:"details" toml:"details"`
+	Weight  float64 `yaml:"weight" toml:"weight"`
+	Run     string  `yaml:"run" toml:"run"`
 }
 
+//---------------------------------------------------------------------------------------
+
 // GitConfig contains the configuration for the Git linters.
 type GitConfig struct {
 	// Maximum size of files in bytes tolerated by the 'git-no-big-files' linter
 	// Default is 10 MB
 	MaxFileSize uint64 `yaml:"maxFileSize" toml:"maxFileSize"`
 }
 
+//---------------------------------------------------------------------------------------
+
 // CodeQualityConfig contains the configuration for the CQ linters used in the Code Quality category
 type CodeQualityConfig struct {
 	// Defines all code linters to use in the Code Quality category
 	Linters []string `yaml:"linters" toml:"linters"`
 }
 
+//---------------------------------------------------------------------------------------
+
 // TestingConfig contains the configuration for the rules in the Testing category.
 type TestingConfig struct {
 	// Filename of the project's test execution report, either absolute or relative to the project's root.
@@ -88,9 +106,16 @@ type TestCoverageTargets struct {
 
 func Default() *Config {
 	return &Config{
-		Rules:       RuleConfig{Disabled: []string{}},
-		Git:         GitConfig{MaxFileSize: 10_000_000}, // 10 MB
-		CodeQuality: CodeQualityConfig{Linters: []string{"pylint", "mypy", "black", "isort", "bandit"}},
+		Rules: RuleConfig{
+			Disabled: []string{},
+			Custom:   []CustomRule{},
+		},
+		Git: GitConfig{
+			MaxFileSize: 10_000_000, // 10 MB
+		},
+		CodeQuality: CodeQualityConfig{
+			Linters: []string{"pylint", "mypy", "black", "isort", "bandit"},
+		},
 		Testing: TestingConfig{
 			Targets: TestingTargets{
 				Minimum: 1,