From a8ffc316a462ea07a5e4cfbfb3df24eb0f9342eb Mon Sep 17 00:00:00 2001 From: David Brownman Date: Fri, 28 Jul 2023 00:44:39 -0700 Subject: [PATCH] Add `test` command to run any unit test (#1092) This commit adds a `test` command that allows the student to run the tests for an exercise without knowing the track-specific test command. This makes it easier for the student to get started. For debugging and education purposes, we print the command that is used to run the tests. --- CHANGELOG.md | 2 + cmd/cmd_test.go | 14 +- cmd/test.go | 84 +++++++++ workspace/exercise_config.go | 57 ++++++ workspace/exercise_config_test.go | 98 ++++++++++ workspace/test_configurations.go | 252 ++++++++++++++++++++++++++ workspace/test_configurations_test.go | 103 +++++++++++ 7 files changed, 604 insertions(+), 6 deletions(-) create mode 100644 cmd/test.go create mode 100644 workspace/exercise_config.go create mode 100644 workspace/exercise_config_test.go create mode 100644 workspace/test_configurations.go create mode 100644 workspace/test_configurations_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 110c6fe80..1c84fa0a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The exercism CLI follows [semantic versioning](http://semver.org/). ---------------- ## Next Release +* [#1092](https://github.com/exercism/cli/pull/1092) Add `exercism test` command to run the unit tests for nearly any track (inspired by [universal-test-runner](https://github.com/xavdid/universal-test-runner)) - [@xavdid] * **Your contribution here** ## v3.1.0 (2022-10-04) @@ -489,5 +490,6 @@ All changes by [@msgehard] [@sfairchild]: https://github.com/sfairchild [@simonjefford]: https://github.com/simonjefford [@srt32]: https://github.com/srt32 +[@xavdid]: https://github.com/xavdid [@williandrade]: https://github.com/williandrade [@zabawaba99]: https://github.com/zabawaba99 diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 76ed0634e..fee08cd98 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -26,12 +26,14 @@ const cfgHomeKey = "EXERCISM_CONFIG_HOME" // test, call the command by calling Execute on the App. // // Example: -// cmdTest := &CommandTest{ -// Cmd: myCmd, -// InitFn: initMyCmd, -// Args: []string{"fakeapp", "mycommand", "arg1", "--flag", "value"}, -// MockInteractiveResponse: "first-input\nsecond\n", -// } +// +// cmdTest := &CommandTest{ +// Cmd: myCmd, +// InitFn: initMyCmd, +// Args: []string{"fakeapp", "mycommand", "arg1", "--flag", "value"}, +// MockInteractiveResponse: "first-input\nsecond\n", +// } +// // cmdTest.Setup(t) // defer cmdTest.Teardown(t) // ... diff --git a/cmd/test.go b/cmd/test.go new file mode 100644 index 000000000..8f5b54e35 --- /dev/null +++ b/cmd/test.go @@ -0,0 +1,84 @@ +package cmd + +import ( + "fmt" + "log" + "os" + "os/exec" + "strings" + + "github.com/exercism/cli/workspace" + "github.com/spf13/cobra" +) + +var testCmd = &cobra.Command{ + Use: "test", + Aliases: []string{"t"}, + Short: "Run the exercise's tests.", + Long: `Run the exercise's tests. + + Run this command in an exercise's root directory.`, + RunE: func(cmd *cobra.Command, args []string) error { + return runTest(args) + }, +} + +func runTest(args []string) error { + track, err := getTrack() + if err != nil { + return err + } + + testConf, ok := workspace.TestConfigurations[track] + + if !ok { + return fmt.Errorf("the \"%s\" track does not yet support running tests using the Exercism CLI. Please see HELP.md for testing instructions", track) + } + + command, err := testConf.GetTestCommand() + if err != nil { + return err + } + cmdParts := strings.Split(command, " ") + + // pass args/flags to this command down to the test handler + if len(args) > 0 { + cmdParts = append(cmdParts, args...) + } + + fmt.Printf("Running tests via `%s`\n\n", strings.Join(cmdParts, " ")) + exerciseTestCmd := exec.Command(cmdParts[0], cmdParts[1:]...) + + // pipe output directly out, preserving any color + exerciseTestCmd.Stdout = os.Stdout + exerciseTestCmd.Stderr = os.Stderr + + err = exerciseTestCmd.Run() + if err != nil { + // unclear what other errors would pop up here, but it pays to be defensive + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode := exitErr.ExitCode() + // if subcommand returned a non-zero exit code, exit with the same + os.Exit(exitCode) + } else { + log.Fatalf("Failed to get error from failed subcommand: %v", err) + } + } + return nil +} + +func getTrack() (string, error) { + metadata, err := workspace.NewExerciseMetadata(".") + if err != nil { + return "", err + } + if metadata.Track == "" { + return "", fmt.Errorf("no track found in exercise metadata") + } + + return metadata.Track, nil +} + +func init() { + RootCmd.AddCommand(testCmd) +} diff --git a/workspace/exercise_config.go b/workspace/exercise_config.go new file mode 100644 index 000000000..9b0164ebb --- /dev/null +++ b/workspace/exercise_config.go @@ -0,0 +1,57 @@ +package workspace + +import ( + "encoding/json" + "errors" + "io/ioutil" + "path/filepath" +) + +const configFilename = "config.json" + +var configFilepath = filepath.Join(ignoreSubdir, configFilename) + +// ExerciseConfig contains exercise metadata. +// Note: we only use a subset of its fields +type ExerciseConfig struct { + Files struct { + Solution []string `json:"solution"` + Test []string `json:"test"` + } `json:"files"` +} + +// NewExerciseConfig reads exercise metadata from a file in the given directory. +func NewExerciseConfig(dir string) (*ExerciseConfig, error) { + b, err := ioutil.ReadFile(filepath.Join(dir, configFilepath)) + if err != nil { + return nil, err + } + var config ExerciseConfig + if err := json.Unmarshal(b, &config); err != nil { + return nil, err + } + + return &config, nil +} + +// GetTestFiles finds returns the names of the file(s) that hold unit tests for this exercise, if any +func (c *ExerciseConfig) GetSolutionFiles() ([]string, error) { + result := c.Files.Solution + if result == nil { + // solution file(s) key was missing in config json, which is an error when calling this fuction + return []string{}, errors.New("no `files.solution` key in your `config.json`. Was it removed by mistake?") + } + + return result, nil +} + +// GetTestFiles finds returns the names of the file(s) that hold unit tests for this exercise, if any +func (c *ExerciseConfig) GetTestFiles() ([]string, error) { + result := c.Files.Test + if result == nil { + // test file(s) key was missing in config json, which is an error when calling this fuction + return []string{}, errors.New("no `files.test` key in your `config.json`. Was it removed by mistake?") + } + + return result, nil +} diff --git a/workspace/exercise_config_test.go b/workspace/exercise_config_test.go new file mode 100644 index 000000000..d07264c0d --- /dev/null +++ b/workspace/exercise_config_test.go @@ -0,0 +1,98 @@ +package workspace + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExerciseConfig(t *testing.T) { + dir, err := ioutil.TempDir("", "exercise_config") + assert.NoError(t, err) + defer os.RemoveAll(dir) + + err = os.Mkdir(filepath.Join(dir, ".exercism"), os.ModePerm) + assert.NoError(t, err) + + f, err := os.Create(filepath.Join(dir, ".exercism", "config.json")) + assert.NoError(t, err) + defer f.Close() + + _, err = f.WriteString(`{ "blurb": "Learn about the basics of Ruby by following a lasagna recipe.", "authors": ["iHiD", "pvcarrera"], "files": { "solution": ["lasagna.rb"], "test": ["lasagna_test.rb"], "exemplar": [".meta/exemplar.rb"] } } `) + assert.NoError(t, err) + + ec, err := NewExerciseConfig(dir) + assert.NoError(t, err) + + assert.Equal(t, ec.Files.Solution, []string{"lasagna.rb"}) + solutionFiles, err := ec.GetSolutionFiles() + assert.NoError(t, err) + assert.Equal(t, solutionFiles, []string{"lasagna.rb"}) + + assert.Equal(t, ec.Files.Test, []string{"lasagna_test.rb"}) + testFiles, err := ec.GetTestFiles() + assert.NoError(t, err) + assert.Equal(t, testFiles, []string{"lasagna_test.rb"}) +} + +func TestExerciseConfigNoTestKey(t *testing.T) { + dir, err := ioutil.TempDir("", "exercise_config") + assert.NoError(t, err) + defer os.RemoveAll(dir) + + err = os.Mkdir(filepath.Join(dir, ".exercism"), os.ModePerm) + assert.NoError(t, err) + + f, err := os.Create(filepath.Join(dir, ".exercism", "config.json")) + assert.NoError(t, err) + defer f.Close() + + _, err = f.WriteString(`{ "blurb": "Learn about the basics of Ruby by following a lasagna recipe.", "authors": ["iHiD", "pvcarrera"], "files": { "exemplar": [".meta/exemplar.rb"] } } `) + assert.NoError(t, err) + + ec, err := NewExerciseConfig(dir) + assert.NoError(t, err) + + _, err = ec.GetSolutionFiles() + assert.Error(t, err, "no `files.solution` key in your `config.json`") + _, err = ec.GetTestFiles() + assert.Error(t, err, "no `files.test` key in your `config.json`") +} + +func TestMissingExerciseConfig(t *testing.T) { + dir, err := ioutil.TempDir("", "exercise_config") + assert.NoError(t, err) + defer os.RemoveAll(dir) + + _, err = NewExerciseConfig(dir) + assert.Error(t, err) + // any assertions about this error message have to work across all platforms, so be vague + // unix: ".exercism/config.json: no such file or directory" + // windows: "open .exercism\config.json: The system cannot find the path specified." + assert.Contains(t, err.Error(), filepath.Join(".exercism", "config.json:")) +} + +func TestInvalidExerciseConfig(t *testing.T) { + dir, err := ioutil.TempDir("", "exercise_config") + assert.NoError(t, err) + defer os.RemoveAll(dir) + + err = os.Mkdir(filepath.Join(dir, ".exercism"), os.ModePerm) + assert.NoError(t, err) + + f, err := os.Create(filepath.Join(dir, ".exercism", "config.json")) + assert.NoError(t, err) + defer f.Close() + + // invalid JSON + _, err = f.WriteString(`{ "blurb": "Learn about the basics of Ruby by following a lasagna recipe.", "authors": ["iHiD", "pvcarr `) + assert.NoError(t, err) + + _, err = NewExerciseConfig(dir) + assert.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "unexpected end of JSON input")) +} diff --git a/workspace/test_configurations.go b/workspace/test_configurations.go new file mode 100644 index 000000000..32c6fbc3a --- /dev/null +++ b/workspace/test_configurations.go @@ -0,0 +1,252 @@ +package workspace + +import ( + "fmt" + "runtime" + "strings" +) + +type TestConfiguration struct { + // The static portion of the test Command, which will be run for every test on this track. Examples include `cargo test` or `go test`. + // Might be empty if there are platform-specific versions + Command string + + // Windows-specific test command. Mostly relevant for tests wrapped by shell invocations. Falls back to `Command` if we're not running windows or this is empty. + WindowsCommand string +} + +func (c *TestConfiguration) GetTestCommand() (string, error) { + var cmd string + if runtime.GOOS == "windows" && c.WindowsCommand != "" { + cmd = c.WindowsCommand + } else { + cmd = c.Command + } + + // pre-declare these so we can conditionally initialize them + var exerciseConfig *ExerciseConfig + var err error + + if strings.Contains(cmd, "{{") { + // only read exercise's config.json if we need it + exerciseConfig, err = NewExerciseConfig(".") + if err != nil { + return "", err + } + } + + if strings.Contains(cmd, "{{solution_files}}") { + if exerciseConfig == nil { + return "", fmt.Errorf("exerciseConfig not initialize before use") + } + solutionFiles, err := exerciseConfig.GetSolutionFiles() + if err != nil { + return "", err + } + cmd = strings.ReplaceAll(cmd, "{{solution_files}}", strings.Join(solutionFiles, " ")) + } + if strings.Contains(cmd, "{{test_files}}") { + if exerciseConfig == nil { + return "", fmt.Errorf("exerciseConfig not initialize before use") + } + testFiles, err := exerciseConfig.GetTestFiles() + if err != nil { + return "", err + } + cmd = strings.ReplaceAll(cmd, "{{test_files}}", strings.Join(testFiles, " ")) + } + + return cmd, nil +} + +// some tracks aren't (or won't be) implemented; every track is listed either way +var TestConfigurations = map[string]TestConfiguration{ + "8th": { + Command: "bash tester.sh", + WindowsCommand: "tester.bat", + }, + // abap: tests are run via "ABAP Development Tools", not the CLI + "awk": { + Command: "bats {{test_files}}", + }, + "ballerina": { + Command: "bal test", + }, + "bash": { + Command: "bats {{test_files}}", + }, + "c": { + Command: "make", + }, + "cfml": { + Command: "box task run TestRunner", + }, + "clojure": { + // chosen because the docs recommend `clj` by default and `lein` as optional + Command: "clj -X:test", + }, + "cobol": { + Command: "bash test.sh", + WindowsCommand: "pwsh test.ps1", + }, + "coffeescript": { + Command: "jasmine-node --coffee {{test_files}}", + }, + // common-lisp: tests are loaded into a "running Lisp implementation", not the CLI directly + "cpp": { + Command: "make", + }, + "crystal": { + Command: "crystal spec", + }, + "csharp": { + Command: "dotnet test", + }, + "d": { + // this always works even if the user installed DUB + Command: "dmd source/*.d -de -w -main -unittest", + }, + "dart": { + Command: "dart test", + }, + // delphi: tests are run via IDE + "elixir": { + Command: "mix test", + }, + "elm": { + Command: "elm-test", + }, + "emacs-lisp": { + Command: "emacs -batch -l ert -l *-test.el -f ert-run-tests-batch-and-exit", + }, + "erlang": { + Command: "rebar3 eunit", + }, + "fortran": { + Command: "make", + }, + "fsharp": { + Command: "dotnet test", + }, + "gleam": { + Command: "gleam test", + }, + "go": { + Command: "go test", + }, + "groovy": { + Command: "gradle test", + }, + "haskell": { + Command: "stack test", + }, + "java": { + Command: "gradle test", + }, + "javascript": { + Command: "npm run test", + }, + "jq": { + Command: "bats {{test_files}}", + }, + "julia": { + Command: "julia runtests.jl", + }, + "kotlin": { + Command: "./gradlew test", + WindowsCommand: "gradlew.bat test", + }, + "lfe": { + Command: "make test", + }, + "lua": { + Command: "busted", + }, + "mips": { + Command: "java -jar /path/to/mars.jar nc runner.mips impl.mips", + }, + "nim": { + Command: "nim r {{test_files}}", + }, + // objective-c: tests are run via XCode. There's a CLI option (ruby gem `objc`), but the docs note that this is an inferior experience + "ocaml": { + Command: "make", + }, + "perl5": { + Command: "prove .", + }, + // pharo-smalltalk: tests are run via IDE + "php": { + Command: "phpunit {{test_files}}", + }, + // plsql: test are run via a "mounted oracle db" + "powershell": { + Command: "Invoke-Pester", + }, + "prolog": { + Command: "swipl -f {{solution_files}} -s {{test_files}} -g run_tests,halt -t 'halt(1)'", + }, + "purescript": { + Command: "spago test", + }, + "python": { + Command: "python3 -m pytest -o markers=task {{test_files}}", + }, + "r": { + Command: "Rscript {{test_files}}", + }, + "racket": { + Command: "raco test {{test_files}}", + }, + "raku": { + Command: "prove6 {{test_files}}", + }, + "reasonml": { + Command: "npm run test", + }, + "red": { + Command: "red {{test_files}}", + }, + "ruby": { + Command: "ruby {{test_files}}", + }, + "rust": { + Command: "cargo test --", + }, + "scala": { + Command: "sbt test", + }, + // scheme: docs present 2 equally valid test methods (`make chez` and `make guile`). So I wasn't sure which to pick + "sml": { + Command: "poly -q --use {{test_files}}", + }, + "swift": { + Command: "swift test", + }, + "tcl": { + Command: "tclsh {{test_files}}", + }, + "typescript": { + Command: "yarn test", + }, + // unison: tests are run from an active UCM session + "vbnet": { + Command: "dotnet test", + }, + // vimscript: tests are run from inside a vim session + "vlang": { + Command: "v -stats test run_test.v", + }, + "wasm": { + Command: "npm run test", + }, + "wren": { + Command: "wrenc {{test_files}}", + }, + "x86-64-assembly": { + Command: "make", + }, + "zig": { + Command: "zig test {{test_files}}", + }, +} diff --git a/workspace/test_configurations_test.go b/workspace/test_configurations_test.go new file mode 100644 index 000000000..97ff40902 --- /dev/null +++ b/workspace/test_configurations_test.go @@ -0,0 +1,103 @@ +package workspace + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetCommand(t *testing.T) { + testConfig, ok := TestConfigurations["elixir"] + assert.True(t, ok, "unexpectedly unable to find elixir test config") + + cmd, err := testConfig.GetTestCommand() + assert.NoError(t, err) + + assert.Equal(t, cmd, "mix test") +} + +func TestWindowsCommands(t *testing.T) { + testConfig, ok := TestConfigurations["cobol"] + assert.True(t, ok, "unexpectedly unable to find cobol test config") + + cmd, err := testConfig.GetTestCommand() + assert.NoError(t, err) + + if runtime.GOOS == "windows" { + assert.Contains(t, cmd, ".ps1") + assert.NotContains(t, cmd, ".sh") + } else { + assert.Contains(t, cmd, ".sh") + assert.NotContains(t, cmd, ".ps1") + } +} + +func TestGetCommandMissingConfig(t *testing.T) { + testConfig, ok := TestConfigurations["ruby"] + assert.True(t, ok, "unexpectedly unable to find ruby test config") + + _, err := testConfig.GetTestCommand() + assert.Error(t, err) + // any assertions about this error message have to work across all platforms, so be vague + // unix: ".exercism/config.json: no such file or directory" + // windows: "open .exercism\config.json: The system cannot find the path specified." + assert.Contains(t, err.Error(), filepath.Join(".exercism", "config.json:")) +} + +func TestIncludesSolutionAndTestFilesInCommand(t *testing.T) { + testConfig, ok := TestConfigurations["prolog"] + assert.True(t, ok, "unexpectedly unable to find prolog test config") + + // this creates a config file in the test directory and removes it + dir := filepath.Join(".", ".exercism") + defer os.RemoveAll(dir) + err := os.Mkdir(dir, os.ModePerm) + assert.NoError(t, err) + + f, err := os.Create(filepath.Join(dir, "config.json")) + assert.NoError(t, err) + defer f.Close() + + _, err = f.WriteString(`{ "blurb": "Learn about the basics of Prolog by following a lasagna recipe.", "authors": ["iHiD", "pvcarrera"], "files": { "solution": ["lasagna.pl"], "test": ["lasagna_tests.plt"] } } `) + assert.NoError(t, err) + + cmd, err := testConfig.GetTestCommand() + assert.NoError(t, err) + assert.Equal(t, cmd, "swipl -f lasagna.pl -s lasagna_tests.plt -g run_tests,halt -t 'halt(1)'") +} + +func TestIncludesTestFilesInCommand(t *testing.T) { + testConfig, ok := TestConfigurations["ruby"] + assert.True(t, ok, "unexpectedly unable to find ruby test config") + + // this creates a config file in the test directory and removes it + dir := filepath.Join(".", ".exercism") + defer os.RemoveAll(dir) + err := os.Mkdir(dir, os.ModePerm) + assert.NoError(t, err) + + f, err := os.Create(filepath.Join(dir, "config.json")) + assert.NoError(t, err) + defer f.Close() + + _, err = f.WriteString(`{ "blurb": "Learn about the basics of Ruby by following a lasagna recipe.", "authors": ["iHiD", "pvcarrera"], "files": { "solution": ["lasagna.rb"], "test": ["lasagna_test.rb", "some_other_file.rb"], "exemplar": [".meta/exemplar.rb"] } } `) + assert.NoError(t, err) + + cmd, err := testConfig.GetTestCommand() + assert.NoError(t, err) + assert.Equal(t, cmd, "ruby lasagna_test.rb some_other_file.rb") +} + +func TestRustHasTrailingDashes(t *testing.T) { + testConfig, ok := TestConfigurations["rust"] + assert.True(t, ok, "unexpectedly unable to find rust test config") + + cmd, err := testConfig.GetTestCommand() + assert.NoError(t, err) + + assert.True(t, strings.HasSuffix(cmd, "--"), "rust's test command should have trailing dashes") +}