Use toml-test to generate tests (#911)

Fixes: #909

Author: mpldr

Diff for: .gitignore

@@ -3,4 +3,5 @@ fuzz/
 cmd/tomll/tomll
 cmd/tomljson/tomljson
 cmd/tomltestgen/tomltestgen
-dist
+dist
+tests/

Diff for: cmd/tomltestgen/main.go

@@ -7,17 +7,13 @@
 package main
 
 import (
-	"archive/zip"
 	"bytes"
 	"flag"
 	"fmt"
 	"go/format"
-	"io"
-	"io/ioutil"
 	"log"
-	"net/http"
 	"os"
-	"regexp"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"text/template"
@@ -64,30 +60,6 @@ const srcTemplate = "// Generated by tomltestgen for toml-test ref {{.Ref}} on {
 	"}\n" +
 	"{{end}}\n"
 
-func downloadTmpFile(url string) string {
-	log.Println("starting to download file from", url)
-	resp, err := http.Get(url)
-	if err != nil {
-		panic(err)
-	}
-	defer resp.Body.Close()
-
-	tmpfile, err := ioutil.TempFile("", "toml-test-*.zip")
-	if err != nil {
-		panic(err)
-	}
-	defer tmpfile.Close()
-
-	copiedLen, err := io.Copy(tmpfile, resp.Body)
-	if err != nil {
-		panic(err)
-	}
-	if resp.ContentLength > 0 && copiedLen != resp.ContentLength {
-		panic(fmt.Errorf("copied %d bytes, request body had %d", copiedLen, resp.ContentLength))
-	}
-	return tmpfile.Name()
-}
-
 func kebabToCamel(kebab string) string {
 	camel := ""
 	nextUpper := true
@@ -107,19 +79,6 @@ func kebabToCamel(kebab string) string {
 	return camel
 }
 
-func readFileFromZip(f *zip.File) string {
-	reader, err := f.Open()
-	if err != nil {
-		panic(err)
-	}
-	defer reader.Close()
-	bytes, err := ioutil.ReadAll(reader)
-	if err != nil {
-		panic(err)
-	}
-	return string(bytes)
-}
-
 func templateGoStr(input string) string {
 	return strconv.Quote(input)
 }
@@ -138,61 +97,57 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
-	url := "https://codeload.github.com/BurntSushi/toml-test/zip/" + *ref
-	resultFile := downloadTmpFile(url)
-	defer os.Remove(resultFile)
-	log.Println("file written to", resultFile)
-
-	zipReader, err := zip.OpenReader(resultFile)
-	if err != nil {
-		panic(err)
-	}
-	defer zipReader.Close()
-
 	collection := testsCollection{
 		Ref:       *ref,
 		Timestamp: time.Now().Format(time.RFC3339),
 	}
 
-	zipFilesMap := map[string]*zip.File{}
+	dirContent, _ := filepath.Glob("tests/invalid/**/*.toml")
+	for _, f := range dirContent {
+		filename := strings.TrimPrefix(f, "tests/valid/")
+		name := kebabToCamel(strings.TrimSuffix(filename, ".toml"))
 
-	for _, f := range zipReader.File {
-		zipFilesMap[f.Name] = f
+		log.Printf("> [%s] %s\n", "invalid", name)
+
+		tomlContent, err := os.ReadFile(f)
+		if err != nil {
+			fmt.Printf("failed to read test file: %s\n", err)
+			os.Exit(1)
+		}
+
+		collection.Invalid = append(collection.Invalid, invalid{
+			Name:  name,
+			Input: string(tomlContent),
+		})
+		collection.Count++
 	}
 
-	testFileRegexp := regexp.MustCompile(`([^/]+/tests/(valid|invalid)/(.+))\.(toml)`)
-	for _, f := range zipReader.File {
-		groups := testFileRegexp.FindStringSubmatch(f.Name)
-		if len(groups) > 0 {
-			name := kebabToCamel(groups[3])
-			testType := groups[2]
-
-			log.Printf("> [%s] %s\n", testType, name)
-
-			tomlContent := readFileFromZip(f)
-
-			switch testType {
-			case "invalid":
-				collection.Invalid = append(collection.Invalid, invalid{
-					Name:  name,
-					Input: tomlContent,
-				})
-				collection.Count++
-			case "valid":
-				baseFilePath := groups[1]
-				jsonFilePath := baseFilePath + ".json"
-				jsonContent := readFileFromZip(zipFilesMap[jsonFilePath])
-
-				collection.Valid = append(collection.Valid, valid{
-					Name:    name,
-					Input:   tomlContent,
-					JsonRef: jsonContent,
-				})
-				collection.Count++
-			default:
-				panic(fmt.Sprintf("unknown test type: %s", testType))
-			}
+	dirContent, _ = filepath.Glob("tests/valid/**/*.toml")
+	for _, f := range dirContent {
+		filename := strings.TrimPrefix(f, "tests/valid/")
+		name := kebabToCamel(strings.TrimSuffix(filename, ".toml"))
+
+		log.Printf("> [%s] %s\n", "valid", name)
+
+		tomlContent, err := os.ReadFile(f)
+		if err != nil {
+			fmt.Printf("failed reading test file: %s\n", err)
+			os.Exit(1)
+		}
+
+		filename = strings.TrimSuffix(f, ".toml")
+		jsonContent, err := os.ReadFile(filename + ".json")
+		if err != nil {
+			fmt.Printf("failed reading validation json: %s\n", err)
+			os.Exit(1)
 		}
+
+		collection.Valid = append(collection.Valid, valid{
+			Name:    name,
+			Input:   string(tomlContent),
+			JsonRef: string(jsonContent),
+		})
+		collection.Count++
 	}
 
 	log.Printf("Collected %d tests from toml-test\n", collection.Count)
@@ -202,7 +157,7 @@ func main() {
 	}
 	t := template.Must(template.New("src").Funcs(funcMap).Parse(srcTemplate))
 	buf := new(bytes.Buffer)
-	err = t.Execute(buf, collection)
+	err := t.Execute(buf, collection)
 	if err != nil {
 		panic(err)
 	}
@@ -216,7 +171,7 @@ func main() {
 		return
 	}
 
-	err = os.WriteFile(*out, outputBytes, 0644)
+	err = os.WriteFile(*out, outputBytes, 0o644)
 	if err != nil {
 		panic(err)
 	}

Diff for: toml_testgen_support_test.go

@@ -1,3 +1,4 @@
+//go:generate go run github.com/toml-lang/toml-test/cmd/toml-test@master -copy ./tests
 //go:generate go run ./cmd/tomltestgen/main.go -o toml_testgen_test.go
 
 // This is a support file for toml_testgen_test.go