use toml-test to fetch test files

Add dependency on current HEAD of toml-test, since -copy is not yet
released and use it instead of the zip-download to fetch test files.

Fixes: #909

.gitignore

@@ -3,4 +3,5 @@ fuzz/
 cmd/tomll/tomll
 cmd/tomljson/tomljson
 cmd/tomltestgen/tomltestgen
-dist
+dist
+tests/

cmd/tomltestgen/main.go

@@ -17,7 +17,8 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"regexp"
+	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"text/template"
@@ -138,61 +139,56 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
-	url := "https://codeload.github.com/BurntSushi/toml-test/zip/" + *ref
-	resultFile := downloadTmpFile(url)
-	defer os.Remove(resultFile)
-	log.Println("file written to", resultFile)
-
-	zipReader, err := zip.OpenReader(resultFile)
-	if err != nil {
-		panic(err)
-	}
-	defer zipReader.Close()
-
 	collection := testsCollection{
 		Ref:       *ref,
 		Timestamp: time.Now().Format(time.RFC3339),
 	}
 
-	zipFilesMap := map[string]*zip.File{}
+	dirContent, _ := filepath.Glob("tests/invalid/**/*.toml")
+	for _, f := range dirContent {
+		filename := path.Base(f)
+		name := kebabToCamel(strings.TrimSuffix(filename, ".toml"))
+
+		log.Printf("> [%s] %s\n", "invalid", name)
 
-	for _, f := range zipReader.File {
-		zipFilesMap[f.Name] = f
+		tomlContent, err := os.ReadFile(f)
+		if err != nil {
+			fmt.Printf("failed to read test file: %s\n", err)
+			os.Exit(1)
+		}
+		collection.Invalid = append(collection.Invalid, invalid{
+			Name:  name,
+			Input: string(tomlContent),
+		})
+		collection.Count++
 	}
 
-	testFileRegexp := regexp.MustCompile(`([^/]+/tests/(valid|invalid)/(.+))\.(toml)`)
-	for _, f := range zipReader.File {
-		groups := testFileRegexp.FindStringSubmatch(f.Name)
-		if len(groups) > 0 {
-			name := kebabToCamel(groups[3])
-			testType := groups[2]
-
-			log.Printf("> [%s] %s\n", testType, name)
-
-			tomlContent := readFileFromZip(f)
-
-			switch testType {
-			case "invalid":
-				collection.Invalid = append(collection.Invalid, invalid{
-					Name:  name,
-					Input: tomlContent,
-				})
-				collection.Count++
-			case "valid":
-				baseFilePath := groups[1]
-				jsonFilePath := baseFilePath + ".json"
-				jsonContent := readFileFromZip(zipFilesMap[jsonFilePath])
-
-				collection.Valid = append(collection.Valid, valid{
-					Name:    name,
-					Input:   tomlContent,
-					JsonRef: jsonContent,
-				})
-				collection.Count++
-			default:
-				panic(fmt.Sprintf("unknown test type: %s", testType))
-			}
+	dirContent, _ = filepath.Glob("tests/valid/**/*.toml")
+	for _, f := range dirContent {
+		filename := path.Base(f)
+		name := kebabToCamel(strings.TrimSuffix(filename, ".toml"))
+
+		log.Printf("> [%s] %s\n", "valid", name)
+
+		tomlContent, err := os.ReadFile(f)
+		if err != nil {
+			fmt.Printf("failed reading test file: %s\n", err)
+			os.Exit(1)
 		}
+
+		filename = strings.TrimSuffix(f, ".toml")
+		jsonContent, err := os.ReadFile(filename + ".json")
+		if err != nil {
+			fmt.Printf("failed reading validation json: %s\n", err)
+			os.Exit(1)
+		}
+
+		collection.Valid = append(collection.Valid, valid{
+			Name:    name,
+			Input:   string(tomlContent),
+			JsonRef: string(jsonContent),
+		})
+		collection.Count++
 	}
 
 	log.Printf("Collected %d tests from toml-test\n", collection.Count)
@@ -202,7 +198,7 @@ func main() {
 	}
 	t := template.Must(template.New("src").Funcs(funcMap).Parse(srcTemplate))
 	buf := new(bytes.Buffer)
-	err = t.Execute(buf, collection)
+	err := t.Execute(buf, collection)
 	if err != nil {
 		panic(err)
 	}
@@ -216,7 +212,7 @@ func main() {
 		return
 	}
 
-	err = os.WriteFile(*out, outputBytes, 0644)
+	err = os.WriteFile(*out, outputBytes, 0o644)
 	if err != nil {
 		panic(err)
 	}

go.mod

@@ -2,4 +2,7 @@ module github.com/pelletier/go-toml/v2
 
 go 1.16
 
-require github.com/stretchr/testify v1.8.4
+require (
+	github.com/stretchr/testify v1.8.4
+	github.com/toml-lang/toml-test v1.4.1-0.20231018175634-23c30e9eb082
+)

go.sum

@@ -1,3 +1,5 @@
+github.com/BurntSushi/toml v1.3.3-0.20230930221951-d17285a02783 h1:q97mwa1LRDI6nt31wvg+nEVzHdgifAkGzXBIzS2KCEk=
+github.com/BurntSushi/toml v1.3.3-0.20230930221951-d17285a02783/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -10,8 +12,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/toml-lang/toml-test v1.4.1-0.20231018175634-23c30e9eb082 h1:PAob6UNFFl87cEmgb+SUnfdw8SHylj+E3yd62PFUWFY=
+github.com/toml-lang/toml-test v1.4.1-0.20231018175634-23c30e9eb082/go.mod h1:wgp654eXwYoTjStgl444r36NI3q1hyMe6fIXrzyTSVw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+zgo.at/zli v0.0.0-20231011155724-865ee344d4b3 h1:c6mOqJByLZGs0hmCK0Pr+XuDU6XoNcccUROX/8nPP6g=
+zgo.at/zli v0.0.0-20231011155724-865ee344d4b3/go.mod h1:HLAc12TjNGT+VRXr76JnsNE3pbooQtwKWhX+RlDjQ2Y=

toml_testgen_support_test.go

@@ -1,3 +1,4 @@
+//go:generate go run github.com/toml-lang/toml-test/cmd/toml-test -copy ./tests
 //go:generate go run ./cmd/tomltestgen/main.go -o toml_testgen_test.go
 
 // This is a support file for toml_testgen_test.go

tools.go

@@ -0,0 +1,7 @@
+//go:build tools
+
+package toml
+
+import (
+	_ "github.com/toml-lang/toml-test/cmd/toml-test"
+)