Skip to content

Commit b260fc9

Browse files
cpucpu
committed
vectorlint: lint test group IDs/count
Lifts out the schema linting specific bits to their own func and adds a new func for checking that: 1. The "tcId" property of each test is unique within the vector file. 2. The "numberOfTests" property matches the number of tests found in the vector.
1 parent 909346b commit b260fc9

File tree

1 file changed

+147
-87
lines changed

1 file changed

+147
-87
lines changed

‎tools/vectorlint/main.go

+147-87
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,14 @@ import (
1616
"strings"
1717
)
1818

19+
var (
20+
schemaDirectory = flag.String("schemas-dir", "schemas", "directory containing schema files")
21+
vectorsDirectories = flag.String("vectors-dir", "testvectors_v1,testvectors", "comma separated directories containing vector files")
22+
vectorFilter = flag.String("vector-filter", "", "only validate vector files matching the provided pattern")
23+
vectorRegex *regexp.Regexp
24+
)
25+
1926
func main() {
20-
schemaDirectory := flag.String("schemas-dir", "schemas", "directory containing schema files")
21-
vectorsDirectories := flag.String("vectors-dir", "testvectors_v1,testvectors", "comma separated directories containing vector files")
22-
vectorFilter := flag.String("vector-filter", "", "only validate vector files matching the provided pattern")
2327

2428
flag.Parse()
2529

@@ -28,7 +32,6 @@ func main() {
2832
log.Printf("reading schemas from %q\n", *schemaDirectory)
2933
log.Printf("reading vectors from %q\n", vectorDirectoryParts)
3034

31-
var vectorRegex *regexp.Regexp
3235
if *vectorFilter != "" {
3336
vectorRegex = regexp.MustCompile(*vectorFilter)
3437
log.Printf("filtering vectors with %q\n", *vectorFilter)
@@ -41,94 +44,21 @@ func main() {
4144
}
4245
schemaCompiler.AssertFormat() // Opt in to format validation.
4346

44-
var total, valid, invalid, noSchema, ignored int
45-
for _, vectorDir := range vectorDirectoryParts {
46-
err := filepath.WalkDir(vectorDir, func(path string, d fs.DirEntry, err error) error {
47-
if err != nil {
48-
return err
49-
}
50-
51-
if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") {
52-
return nil
53-
}
47+
var results schemaLintResults
5448

55-
if vectorRegex != nil && !vectorRegex.MatchString(d.Name()) {
56-
return nil
57-
}
58-
59-
vectorData, err := os.ReadFile(path)
60-
if err != nil {
61-
return fmt.Errorf("failed to read %s: %w", path, err)
62-
}
63-
64-
total++
65-
66-
var vector struct {
67-
Schema string `json:"schema"`
68-
}
69-
70-
if err := json.Unmarshal(vectorData, &vector); err != nil {
71-
log.Printf("❌ %q: invalid vector JSON data: %s\n", path, err)
72-
invalid++
73-
return nil
74-
}
75-
76-
if vector.Schema == "" {
77-
log.Printf("❌ %q: no schema specified\n", path)
78-
noSchema++
79-
return nil
80-
}
81-
82-
if missingSchemas[vector.Schema] {
83-
log.Printf("⚠️ %q: ignoring missing schema %q\n", path, vector.Schema)
84-
ignored++
85-
return nil
86-
}
87-
88-
schemaPath := filepath.Join(*schemaDirectory, vector.Schema)
89-
if _, err := os.Stat(schemaPath); os.IsNotExist(err) {
90-
log.Printf("❌ %q: referenced schema %q not found\n", path, vector.Schema)
91-
invalid++
92-
return nil
93-
}
94-
95-
schema, err := schemaCompiler.Compile(schemaPath)
96-
if err != nil {
97-
log.Printf("❌ %q: invalid schema %q: %s\n", path, vector.Schema, err)
98-
invalid++
99-
return nil
100-
}
101-
102-
var instance any
103-
if err := json.Unmarshal(vectorData, &instance); err != nil {
104-
log.Printf("❌ %q: invalid vector JSON data: %s\n", path, err)
105-
invalid++
106-
return nil
107-
}
108-
109-
if err := schema.Validate(instance); err != nil {
110-
log.Printf("❌ %q: vector doesn't validate with schema: %s\n", path, err)
111-
invalid++
112-
return nil
113-
}
114-
115-
log.Printf("✅ %q: validates with %q\n", path, vector.Schema)
116-
valid++
117-
return nil
118-
})
119-
if err != nil {
120-
fmt.Printf("Error walking directory: %v\n", err)
121-
os.Exit(1)
49+
for _, vectorDir := range vectorDirectoryParts {
50+
if err := lintVectorDir(schemaCompiler, &results, vectorDir); err != nil {
51+
log.Fatalf("error linting schemas: %v\n", err)
12252
}
12353
}
12454

125-
log.Printf("linted %d vector files\n", total)
126-
log.Printf("valid: %d\n", valid)
127-
log.Printf("invalid: %d\n", invalid)
128-
log.Printf("no schema: %d\n", noSchema)
129-
log.Printf("ignored: %d\n", ignored)
55+
log.Printf("linted %d vector files\n", results.total)
56+
log.Printf("valid: %d\n", results.valid)
57+
log.Printf("invalid: %d\n", results.invalid)
58+
log.Printf("no schema: %d\n", results.noSchema)
59+
log.Printf("ignored: %d\n", results.ignored)
13060

131-
os.Exit(invalid)
61+
os.Exit(results.invalid)
13262
}
13363

13464
var (
@@ -232,3 +162,133 @@ func validateCurve(value any) error {
232162
return fmt.Errorf("invalid EcCurve: unknown curve name: %v", value)
233163
}
234164
}
165+
166+
func lintVectorDir(schemaCompiler *jsonschema.Compiler, results *schemaLintResults, vectorDir string) error {
167+
err := filepath.WalkDir(vectorDir, func(path string, d fs.DirEntry, err error) error {
168+
if err != nil {
169+
return err
170+
}
171+
172+
if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") {
173+
return nil
174+
}
175+
176+
if vectorRegex != nil && !vectorRegex.MatchString(d.Name()) {
177+
return nil
178+
}
179+
180+
results.total++
181+
182+
vectorData, err := os.ReadFile(path)
183+
if err != nil {
184+
return fmt.Errorf("failed to read %s: %w", path, err)
185+
}
186+
187+
if err := lintVectorTestGroups(vectorData, path); err != nil {
188+
log.Printf("❌ %q: %s\n", path, err)
189+
results.invalid++
190+
return nil
191+
}
192+
193+
if err := lintVectorToSchema(schemaCompiler, vectorData, path, results); err != nil {
194+
return err
195+
}
196+
197+
return nil
198+
})
199+
if err != nil {
200+
return fmt.Errorf("error walking directory: %w", err)
201+
}
202+
203+
return nil
204+
}
205+
206+
func lintVectorTestGroups(vectorData []byte, path string) error {
207+
var vector struct {
208+
NumberOfTests int `json:"numberOfTests"`
209+
TestGroups []struct {
210+
Tests []struct {
211+
TcId int `json:"tcId"`
212+
} `json:"tests"`
213+
} `json:"testGroups"`
214+
}
215+
if err := json.Unmarshal(vectorData, &vector); err != nil {
216+
return fmt.Errorf("error decoding vector JSON data for test groups: %w", err)
217+
}
218+
219+
// Within a vector file, test case IDs must be unique.
220+
testCaseIds := make(map[int]struct{})
221+
for _, tg := range vector.TestGroups {
222+
for _, test := range tg.Tests {
223+
if _, ok := testCaseIds[test.TcId]; ok {
224+
return fmt.Errorf("vector %q has duplicate tcId %d", path, test.TcId)
225+
}
226+
testCaseIds[test.TcId] = struct{}{}
227+
}
228+
}
229+
230+
if testCount := len(testCaseIds); testCount != vector.NumberOfTests {
231+
return fmt.Errorf("vector %q declared %d tests in group, had %d", path, vector.NumberOfTests, testCount)
232+
}
233+
234+
return nil
235+
}
236+
237+
func lintVectorToSchema(schemaCompiler *jsonschema.Compiler, vectorData []byte, path string, results *schemaLintResults) error {
238+
var vector struct {
239+
Schema string `json:"schema"`
240+
}
241+
242+
if err := json.Unmarshal(vectorData, &vector); err != nil {
243+
log.Printf("❌ %q: invalid vector JSON data: %s\n", path, err)
244+
results.invalid++
245+
return nil
246+
}
247+
248+
if vector.Schema == "" {
249+
log.Printf("❌ %q: no schema specified\n", path)
250+
results.noSchema++
251+
return nil
252+
}
253+
254+
if missingSchemas[vector.Schema] {
255+
log.Printf("⚠️ %q: ignoring missing schema %q\n", path, vector.Schema)
256+
results.ignored++
257+
return nil
258+
}
259+
260+
schemaPath := filepath.Join(*schemaDirectory, vector.Schema)
261+
if _, err := os.Stat(schemaPath); os.IsNotExist(err) {
262+
log.Printf("❌ %q: referenced schema %q not found\n", path, vector.Schema)
263+
results.invalid++
264+
return nil
265+
}
266+
267+
schema, err := schemaCompiler.Compile(schemaPath)
268+
if err != nil {
269+
log.Printf("❌ %q: invalid schema %q: %s\n", path, vector.Schema, err)
270+
results.invalid++
271+
return nil
272+
}
273+
274+
var instance any
275+
if err := json.Unmarshal(vectorData, &instance); err != nil {
276+
log.Printf("❌ %q: invalid vector JSON data: %s\n", path, err)
277+
results.invalid++
278+
return nil
279+
}
280+
281+
if err := schema.Validate(instance); err != nil {
282+
log.Printf("❌ %q: vector doesn't validate with schema: %s\n", path, err)
283+
results.invalid++
284+
return nil
285+
}
286+
287+
log.Printf("✅ %q: validates with %q\n", path, vector.Schema)
288+
results.valid++
289+
return nil
290+
}
291+
292+
type schemaLintResults struct {
293+
total, valid, invalid, noSchema, ignored int
294+
}

0 commit comments

Comments
 (0)