Add first version of batch testing

Supported features now:

- Creating the batching structure as described in the bug.
- Running the batch with the given parameters

Still missing:

- User should be able to write the brute solution with it's preferred
langauge, not necessarly C++
- "rand.h" is still a dummy library for now and will be added as part of
another bug.

Author: chermehdi

README.md

@@ -124,6 +124,47 @@ type TemplateContext struct {
 ```
 You can access any of the model fields in the template, and make changes accordingly dependending on your preferences.
 - Running the command `egor config set config.templates.{lang} /path/to/template/file` will register the given template and will use it for the given language for future tasks.
+
+## Batch testing
+
+- This one is usefull if you are stuck on a problem, your solution gets `WA` and
+you can't figure out a testcase that will make it break.
+- If you know how to write a brute force solution to the problem, egor can help
+  with running the that solution that you know gives the correct answer against
+  your (optimal) solution, and figure out when will they differ.
+
+### Steps to make a batch test
+- Create a batch configuration
+
+```
+egor batch create --has-brute
+```
+
+Running this will create 3 additional files: 
+
+- `gen.cpp`: which is the random input generator, this include a small random
+  generation library `"rand.h"`, feel free to use it to generate testcases for
+  the given problem.
+- `main_brute.cpp`: which will act as the brute force solution.
+- `rand.h`: the random generation library, contains a bunch of helper methods
+  to generate random input, read the source and docs on public methods for
+  additional info.
+
+Once all the files are filled according to their purpose, you can start running
+your batch test using
+
+```
+egor batch run --tests=10
+```
+
+the parameter `--tests` tells egor how many times it should run your code,
+a `100` is a good start.
+
+If you can't find a failing testcase after this, you can increase the
+`--tests` parameter, or change the generation strategy, (ex: try bigger
+/ smaller numbers), egor will give your program a random seed every time it runs
+the generator, so that it generates a new testcase with every run.
+
 ## Contribution
 
 - Contribution to the project can be done in multiple ways, you can report bugs and issues or you can discuss new features that you like being added in future versions by creating a new issue

commands/batch.go

@@ -1,36 +1,29 @@
 package commands
 
 import (
-	"os"
-	"path"
-	template2 "text/template"
-
+	"errors"
+	"fmt"
 	"github.com/chermehdi/egor/config"
 	"github.com/chermehdi/egor/templates"
+	"github.com/chermehdi/egor/utils"
 	"github.com/fatih/color"
 	"io/ioutil"
+	"log"
+	"math/rand"
+	"os"
+	"path"
+	template2 "text/template"
 
 	"github.com/urfave/cli/v2"
 )
 
-type Batcher struct {
-	judge Judge
-}
-
-func (b *Batcher) Setup() error {
-	b.judge.Setup()
-	return nil
-}
-
-func (b *Batcher) Run() error {
-	return nil
-}
-
-func (b *Batcher) Cleanup() error {
-	b.judge.Cleanup()
-	return nil
-}
+const (
+	RandName  = "rand.h"
+	BruteName = "main_brute.cpp"
+	GenName   = "gen.cpp"
+)
 
+// CreateBatch will create the batch directory structure with the files needed
 func CreateBatch(context *cli.Context) error {
 	configuration, err := config.LoadDefaultConfiguration()
 	if err != nil {
@@ -51,7 +44,7 @@ func CreateBatch(context *cli.Context) error {
 		return nil
 	}
 
-	genPath := path.Join(cwd, "gen.cpp")
+	genPath := path.Join(cwd, GenName)
 	egorMeta.BatchFile = genPath
 	// Create the generator file
 
@@ -68,10 +61,17 @@ func CreateBatch(context *cli.Context) error {
 	if err = compiledTemplate.Execute(genFile, configuration); err != nil {
 		return err
 	}
-	randH := path.Join(cwd, "rand.h")
-	// Create the rand.h file
-	if err = ioutil.WriteFile(randH, []byte(templates.RandH), 0755); err != nil {
-		return err
+	if context.Bool("has-brute") {
+		randH := path.Join(cwd, RandName)
+		// Create the rand.h file
+		if err = ioutil.WriteFile(randH, []byte(templates.RandH), 0755); err != nil {
+			return err
+		}
+
+		bruteH := path.Join(cwd, BruteName)
+		if err = ioutil.WriteFile(bruteH, []byte(templates.BruteH), 0755); err != nil {
+			return err
+		}
 	}
 	// Update the metadata
 	fileName := path.Join(cwd, configuration.ConfigFileName)
@@ -82,7 +82,7 @@ func CreateBatch(context *cli.Context) error {
 	return nil
 }
 
-func RunBatch(context *cli.Context) error {
+func runBatchInternal(context *cli.Context) error {
 	configuration, err := config.LoadDefaultConfiguration()
 	if err != nil {
 		return err
@@ -96,27 +96,182 @@ func RunBatch(context *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	if !egorMeta.HasBatch() {
+		color.Red("The current task does not have a batch setting, did you forget to run egor batch create?")
+		return nil
+	}
+	n := context.Int("tests")
+	runner, found := utils.CreateRunner(egorMeta.TaskLang)
+	// This used to run the generator
+	if !found {
+		color.Red("No task runner found for the task's default language, make sure to not change the egor metafile unless manually unless you know what you are doing!")
+		return errors.New("Task runner not found!")
+	}
+	cppr, _ := utils.CreateRunner("cpp")
 
-	judge, err := NewJudgeFor(egorMeta, configuration, getChecker(context.String("checker")))
+	// Compile the generator
+	c := &utils.ExecutionContext{
+		FileName:   GenName,
+		BinaryName: "gen",
+		Input:      nil,
+		Dir:        cwd,
+	}
+
+	log.Println("Compiling the generator...")
+
+	res, err := cppr.Compile(c)
 	if err != nil {
+		if res.Stderr.String() != "" {
+			color.Red(res.Stderr.String())
+		}
 		return err
 	}
-	if err := judge.Setup(); err != nil {
+
+	log.Println("Finished compiling the generator")
+
+	log.Println("Compiling the solution ...")
+	binName := getBinaryName(egorMeta.TaskLang)
+	// Compile the solution
+	c1 := &utils.ExecutionContext{
+		FileName:   egorMeta.TaskFile,
+		BinaryName: binName,
+		Input:      nil,
+		Dir:        cwd,
+	}
+
+	res, err = runner.Compile(c1)
+	if err != nil {
+		if res.Stderr.String() != "" {
+			color.Red(res.Stderr.String())
+		}
+		return err
+	}
+
+	log.Println("Finished compiling the solution")
+
+	log.Println("Finished compiling the brute force solution")
+	// Compile the brute force solution
+	c2 := &utils.ExecutionContext{
+		FileName:   BruteName,
+		BinaryName: "brute",
+		Input:      nil,
+		Dir:        cwd,
+	}
+
+	res, err = cppr.Compile(c2)
+	if err != nil {
+		if res.Stderr.String() != "" {
+			color.Red(res.Stderr.String())
+		}
 		return err
 	}
-	defer judge.Cleanup()
-	report := newJudgeReport()
 
-	for i, input := range egorMeta.Inputs {
-		output := egorMeta.Outputs[i]
-		caseDescription := NewCaseDescription(input, output, egorMeta.TimeLimit)
-		status := judge.RunTestCase(*caseDescription)
-		report.Add(status, *caseDescription)
+	log.Println("Finished compiling the brute force solution")
+
+	checker := &TokenChecker{}
+
+	for i := 0; i < n; i = i + 1 {
+		color.Green(fmt.Sprintf("Running test %d", i+1))
+
+		log.Println("Running the generator.. ")
+		// Args should contain the random seed to give generator as argument
+		c.Args = fmt.Sprintf("%d", rand.Intn(int(1<<30)))
+
+		// Get the random input
+		res, err = cppr.Run(c)
+		if err != nil {
+			color.Red("Could not run the generator ")
+			if res.Stderr.String() != "" {
+				color.Red(res.Stderr.String())
+			}
+			return err
+		}
+		log.Println("Finished running the generator")
+		log.Printf("Generator output: \n%s\n", res.Stdout.String())
+
+		log.Println("Running the main solution..")
+
+		// making two input copies:
+		//   - one for the brute solution consumer
+		//   - second only used in case of a found diff to print the output to the
+		//   input to the user
+		inC1 := res.Stdout
+		inC2 := res.Stdout
+
+		// Run the main solution
+		// Feed the output of the generator as the main solution's input.
+		c1.Input = &res.Stdout
+		main, err := runner.Run(c1)
+		if err != nil {
+			color.Red("Could not run the solution")
+			if main.Stderr.String() != "" {
+				color.Red(main.Stderr.String())
+			}
+			return err
+		}
+
+		log.Println("Finished running main the solution")
+
+		// Run the main solution
+		// Feed the output of the generator as the brute solution's input.
+		log.Println("Running the brute solution ...")
+
+		c2.Input = &inC1
+		brute, err := cppr.Run(c2)
+		if err != nil {
+			color.Red("Could not run the brute solution")
+			if brute.Stderr.String() != "" {
+				color.Red(brute.Stderr.String())
+			}
+			return err
+		}
+
+		if err := checker.Check(brute.Stdout.String(), main.Stdout.String()); err != nil {
+			color.Red("Found diff: ")
+			color.Red("Input: ")
+			fmt.Println(inC2.String())
+			color.Red("Expected: ")
+			fmt.Println(brute.Stdout.String())
+			color.Red("Got: ")
+			fmt.Println(main.Stdout.String())
+			color.Red(err.Error())
+			return err
+		}
 	}
-	report.Display()
+
+	color.Green("All green, not diff found - Your solution is probably correct!")
 	return nil
 }
 
+// Compute the binary name according to the language.
+func getBinaryName(lang string) string {
+	switch lang {
+	case "cpp":
+		return "sol"
+	case "java":
+		return "Main"
+	case "python":
+		return "main.py"
+	}
+	// should not get here.
+	return ""
+}
+
+// RunBatch will delegate to the internal function to do the actual batch
+// running and it will handle clean up afterwards (delete residual binary
+// files).
+func RunBatch(context *cli.Context) error {
+	err := runBatchInternal(context)
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	os.Remove(path.Join(cwd, "sol"))
+	os.Remove(path.Join(cwd, "brute"))
+	os.Remove(path.Join(cwd, "gen"))
+	return err
+}
+
 var BatchCommand = cli.Command{
 	Name:      "batch",
 	Aliases:   []string{"b"},
@@ -128,12 +283,28 @@ var BatchCommand = cli.Command{
 			Aliases: []string{"c"},
 			Usage:   "Create the template for the batch test",
 			Action:  CreateBatch,
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:    "has-brute",
+					Usage:   "Does this solution have a brute force solution",
+					Aliases: []string{"b"},
+					Value:   true,
+				},
+			},
 		},
 		{
 			Name:    "run",
 			Aliases: []string{"r"},
 			Usage:   "Run the batch test",
 			Action:  RunBatch,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:    "tests",
+					Usage:   "Number of tests to run in the batch round",
+					Aliases: []string{"t"},
+					Value:   100,
+				},
+			},
 		},
 	},
 }

commands/create.go

@@ -109,7 +109,7 @@ func CreateTaskAction(context *cli.Context) error {
 		Output: config.IOType{
 			Type: "stdout",
 		},
-		Languages: nil,
+		Languages: map[string]config.LanguageDescription{},
 	}
 	configuration, err := config.LoadDefaultConfiguration()
 	if err != nil {