-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Starts implementing a linter for the Testing category: user should pr…
…ovide JUnit XML test report and Cobertura-compatible XML coverage report that will be analysed for test completion and coverage. Implements checking test completion stats from JUnit XML
- Loading branch information
Showing
8 changed files
with
218 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,110 @@ | ||
package template | ||
|
||
import ( | ||
"fmt" | ||
"path" | ||
"strings" | ||
|
||
"github.com/joshdk/go-junit" | ||
|
||
"github.com/bvobart/mllint/api" | ||
"github.com/bvobart/mllint/categories" | ||
"github.com/bvobart/mllint/config" | ||
"github.com/bvobart/mllint/utils" | ||
) | ||
|
||
func NewLinter() api.ConfigurableLinter { | ||
return &TestingLinter{} | ||
} | ||
|
||
type TestingLinter struct { | ||
Config config.TestingConfig | ||
} | ||
|
||
func (l *TestingLinter) Name() string { | ||
return categories.Testing.Name | ||
} | ||
|
||
func (l *TestingLinter) Configure(conf *config.Config) error { | ||
l.Config = conf.Testing | ||
return nil | ||
} | ||
|
||
func (l *TestingLinter) Rules() []*api.Rule { | ||
return []*api.Rule{&RuleHasTests, &RuleTestsPass, &RuleTestCoverage, &RuleTestsFolder} | ||
} | ||
|
||
func (l *TestingLinter) LintProject(project api.Project) (api.Report, error) { | ||
report := api.NewReport() | ||
|
||
// TODO: find and count the amount of test files to score RuleHasTests | ||
l.ScoreRuleHasTests(&report, project) | ||
l.ScoreRuleTestsPass(&report, project) | ||
|
||
// TODO: implement the linting for RuleTestCoverage, which checks whether there is a Cobertura XML coverage report and analyses it for test coverage. | ||
// TODO: check whether all test files are in tests folder. | ||
|
||
return report, nil | ||
} | ||
|
||
func (l *TestingLinter) ScoreRuleHasTests(report *api.Report, project api.Project) { | ||
testFiles := project.PythonFiles.Filter(func(filename string) bool { | ||
return strings.HasSuffix(filename, "_test.py") || strings.HasPrefix(path.Base(filename), "test_") | ||
}) | ||
|
||
// there should be at 1 test file per 4 non-test Python files. | ||
report.Scores[RuleHasTests] = 100 * (float64(len(testFiles)) * 4 / float64(len(project.PythonFiles)-len(testFiles))) | ||
} | ||
|
||
func (l *TestingLinter) ScoreRuleTestsPass(report *api.Report, project api.Project) { | ||
if l.Config.Report == "" { | ||
report.Scores[RuleTestsPass] = 0 | ||
report.Details[RuleTestsPass] = "No test report was provided. Please update the `testing.report` setting in your project's `mllint` configuration to specify the path to your project's test report.\n\n" + howToMakeJUnitXML | ||
return | ||
} | ||
|
||
if !utils.FileExists(l.Config.Report) { | ||
report.Scores[RuleTestsPass] = 0 | ||
report.Details[RuleTestsPass] = fmt.Sprintf("A test report was provided, namely `%s`, but this file could not be found. Please update the `testing.report` setting in your project's `mllint` configuration to fix the path to your project's test report.", l.Config.Report) | ||
return | ||
} | ||
|
||
suites, err := junit.IngestFile(l.Config.Report) | ||
if err != nil { | ||
report.Scores[RuleTestsPass] = 0 | ||
report.Details[RuleTestsPass] = fmt.Sprintf(`A test report file `+"`%s`"+` was provided and found, but there was an error parsing the JUnit XML contents: | ||
%s | ||
Please make sure your test report file is a valid JUnit XML file. %s`, l.Config.Report, "```\n"+err.Error()+"\n```", howToMakeJUnitXML) | ||
return | ||
} | ||
|
||
passedTests := 0 | ||
totalTests := 0 | ||
for _, suite := range suites { | ||
totalTests += suite.Totals.Tests | ||
passedTests += suite.Totals.Passed | ||
} | ||
|
||
if totalTests == 0 { | ||
report.Scores[RuleTestsPass] = 0 | ||
report.Details[RuleTestsPass] = fmt.Sprintf(`No tests were run, according to the provided test report file `+"`%s`"+`. Don't be shy, implement some tests!`, l.Config.Report) | ||
} | ||
|
||
score := 100 * float64(passedTests) / float64(totalTests) | ||
report.Scores[RuleTestsPass] = score | ||
if passedTests == totalTests { | ||
report.Details[RuleTestsPass] = fmt.Sprintf("Congratulations, all of your project's %d tests passed!", totalTests) | ||
} else if passedTests == 0 { | ||
report.Details[RuleTestsPass] = fmt.Sprintf("Oh no! What a shame... **None** of your project's %d tests passed! There must be something terribly wrong.", totalTests) | ||
} else if score < 0.25 { | ||
report.Details[RuleTestsPass] = fmt.Sprintf("Oh no! Only **%d** out of **%d** tests passed... That's less than a quarter of all tests in your project...", passedTests, totalTests) | ||
} else if score > 0.75 { | ||
report.Details[RuleTestsPass] = fmt.Sprintf("Hmm, only **%d** out of **%d** of your project's tests passed... That's over three quarter of all tests in your project, but it's not enough: all tests must pass. Good luck fixing the broken tests!", passedTests, totalTests) | ||
} else { | ||
report.Details[RuleTestsPass] = fmt.Sprintf("Oh my, only **%d** out of **%d** of your project's tests passed... You can do better, right? Good luck fixing those tests!", passedTests, totalTests) | ||
} | ||
} | ||
|
||
const howToMakeJUnitXML = "When using `pytest` to run your project's tests, use the `--junitxml=<filename>` option to generate such a test report, e.g.: `pytest --junitxml=tests-report.xml`" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
package template | ||
|
||
import "github.com/bvobart/mllint/api" | ||
|
||
var RuleHasTests = api.Rule{ | ||
Name: "Project has automated tests", | ||
Slug: "testing/has-tests", | ||
Details: "TODO", | ||
Weight: 1, | ||
} | ||
|
||
var RuleTestsPass = api.Rule{ | ||
Name: "Project passes all of its automated tests", | ||
Slug: "testing/pass", | ||
Details: "TODO", | ||
Weight: 1, | ||
} | ||
|
||
var RuleTestCoverage = api.Rule{ | ||
Name: "Project provides a test coverage report", | ||
Slug: "testing/coverage", | ||
Details: "TODO: fill this in with details about what the rule checks, what the reasoning behind it is, where people can get more information on how to implement what this rule checks.", | ||
Weight: 1, | ||
} | ||
|
||
var RuleTestsFolder = api.Rule{ | ||
Name: "Tests should be placed in the tests folder", | ||
Slug: "testing/tests-folder", | ||
Details: "TODO", | ||
Weight: 1, | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters