fixturescript: a tiny scripting engine for specifying test fixtures

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.5.0
+	github.com/hugelgupf/go-shlex v0.0.0-20200702092117-c80c9d0918fa
 	github.com/jackc/pgconn v1.14.1
 	github.com/jackc/pgtype v1.14.0
 	github.com/jackc/pgx/v4 v4.18.0

go.sum

@@ -48,6 +48,8 @@ github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbu
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hugelgupf/go-shlex v0.0.0-20200702092117-c80c9d0918fa h1:s3KPo0nThtvjEamF/aElD4k5jSsBHew3/sgNTnth+2M=
+github.com/hugelgupf/go-shlex v0.0.0-20200702092117-c80c9d0918fa/go.mod h1:I1uW6ymzwsy5TlQgD1bFAghdMgBYqH1qtCeHoZgHMqs=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
 github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=

test/fixturescript/fixturescript.go

@@ -0,0 +1,171 @@
+// Package fixturescript is a small language to declare claircore test fixtures.
+//
+// Fixture scripts are much easier to understand and modify than JSON or gob
+// inputs, and allow for the serialization particulars of any type to be
+// independent of the tests.
+//
+// # Language
+// Each line of a script is parsed into a sequence of space-separated command
+// words using shell quoting rules, with # marking an end-of-line comment.
+//
+// Exact semantics depend on the fixture being constructed, but generally
+// commands are imperative: commands will affect commands that come after.
+// The [CreateIndexReport] example demonstrates how it typically works.
+//
+// # Implementations
+//
+// The [Parse] function is generic, but requires specific conventions for the
+// types passed in because dispatch happens via [reflect]. See the documentation
+// of [Parse] and [Parser].
+package fixturescript
+
+import (
+	"bufio"
+	"encoding"
+	"encoding/json"
+	"fmt"
+	"io"
+	"reflect"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"github.com/hugelgupf/go-shlex"
+)
+
+// Parse ...
+func Parse[T any, Ctx any](out Parser[*T], pc *Ctx, name string, r io.Reader) (*T, error) {
+	fv := reflect.ValueOf(out)
+	// Do some reflect nastiness to make sure "fv" ends up with a pointer in it.
+WalkType:
+	switch fv.Kind() {
+	case reflect.Pointer: // OK
+	case reflect.Interface:
+		fv = fv.Elem().Addr()
+		goto WalkType
+	default:
+		fv = fv.Addr()
+		goto WalkType
+	}
+	// We'll be passing this in to every call, so create it once.
+	pcv := reflect.ValueOf(pc)
+	// Use a static list of prefixes to use when constructing a call.
+	// This allows for shorter names for more common cases.
+	prefixes := []string{"", "Add", "Push", "Pop"}
+
+	// TODO(hank) This function might be sped up by keeping a global cache for
+	// this dispatcher construction, keyed by the "Parser" type.
+	calls := make(map[string]reflect.Value)
+	ft := fv.Type()
+	for i := 0; i < ft.NumMethod(); i++ {
+		m := ft.Method(i)
+		// Disallow a command of the one method we statically know must be here.
+		// It's not a real script environment, there's no way to manipulate the returned value.
+		if m.Name == "Value" {
+			continue
+		}
+		calls[m.Name] = m.Func
+	}
+
+	s := bufio.NewScanner(r)
+	s.Split(bufio.ScanLines)
+	lineNo := 0
+	for s.Scan() {
+		lineNo++
+		line, _, _ := strings.Cut(s.Text(), "#")
+		if len(line) == 0 {
+			continue
+		}
+
+		var cmd string
+		var args []string
+		if i := strings.IndexFunc(line, unicode.IsSpace); i == -1 {
+			cmd = line
+		} else {
+			cmd = line[:i]
+			args = shlex.Split(line[i:])
+		}
+
+		// Slightly odd construction to try all the prefixes:
+		// as soon as one name is valid, jump past the error return.
+		var m reflect.Value
+		var ok bool
+		for _, pre := range prefixes {
+			m, ok = calls[pre+cmd]
+			if ok {
+				goto Call
+			}
+		}
+		return nil, fmt.Errorf("%s:%d: unrecognized command %q", name, lineNo, cmd)
+
+	Call:
+		av := reflect.ValueOf(args)
+		// Next two lines will panic if not following the convention:
+		res := m.Call([]reflect.Value{fv, pcv, av})
+		if errRet := res[0]; !errRet.IsNil() {
+			return nil, fmt.Errorf("%s:%d: command %s: %w", name, lineNo, cmd, errRet.Interface().(error))
+		}
+	}
+	if err := s.Err(); err != nil {
+		return nil, fmt.Errorf("%s: %w", name, err)
+	}
+
+	return out.Value(), nil
+}
+
+// Parser ...
+//
+// There are additional restrictions on values used as a Parser:
+//
+//   - Any exported methods must have a pointer receiver.
+//   - Exported methods must accept the "Ctx" type passed to [Parse] as the first argument,
+//     a slice of strings as the second argument,
+//     and return an [error].
+type Parser[Out any] interface {
+	Value() Out
+}
+
+// AssignToStruct is a helper for writing setter commands.
+//
+// It interprets the "args" array as a key-value pair separated by a "=".
+// If the key is the name of a field, the value is interpreted as the type of
+// the field and assigned to it. Supported types are:
+//
+//   - int64
+//   - int
+//   - string
+//   - encoding.TextUnmarshaler
+//   - json.Unmarshaler
+func AssignToStruct[T any](tgt *T, args []string) (err error) {
+	dv := reflect.ValueOf(tgt).Elem()
+	for _, arg := range args {
+		k, v, ok := strings.Cut(arg, "=")
+		if !ok {
+			return fmt.Errorf("malformed arg: %q", arg)
+		}
+		f := dv.FieldByName(k)
+		if !f.IsValid() {
+			return fmt.Errorf("unknown key: %q", k)
+		}
+		switch x := f.Addr().Interface(); x := x.(type) {
+		case *int64:
+			*x, err = strconv.ParseInt(v, 10, 0)
+		case *int:
+			var tmp int64
+			tmp, err = strconv.ParseInt(v, 10, 0)
+			if err == nil {
+				*x = int(tmp)
+			}
+		case *string:
+			*x = v
+		case encoding.TextUnmarshaler:
+			err = x.UnmarshalText([]byte(v))
+		case json.Unmarshaler:
+			err = x.UnmarshalJSON([]byte(v))
+		}
+		if err != nil {
+			return fmt.Errorf("key %q: bad value %q: %w", k, v, err)
+		}
+	}
+	return nil
+}

test/fixturescript/indexreport.go

@@ -0,0 +1,156 @@
+package fixturescript
+
+import (
+	"errors"
+	"io"
+	"strconv"
+
+	"github.com/quay/claircore"
+)
+
+// TODO(hank) Should this import `claircore`? Might be cycle concerns.
+
+// CreateIndexReport ...
+//
+// Has the following commands:
+//
+//   - AddManifest (sets manifest digest, only allowed once)
+//   - AddLayer (sets current layer digest)
+//   - AddDistribution (sets current distribution)
+//   - ClearDistribution (clears current distribution)
+//   - PushRepository (pushes a repository onto the repository stack)
+//   - PopRepository (pops a repository off the repository stack)
+//   - AddPackage (emits a package using the current Distribution and repository stack)
+func CreateIndexReport(name string, r io.Reader) (*claircore.IndexReport, error) {
+	f := indexReportFixure{
+		IndexReport: &claircore.IndexReport{
+			Packages:      make(map[string]*claircore.Package),
+			Distributions: make(map[string]*claircore.Distribution),
+			Repositories:  make(map[string]*claircore.Repository),
+			Environments:  make(map[string][]*claircore.Environment),
+		},
+	}
+	pc := indexReportCtx{}
+	return Parse(&f, &pc, name, r)
+}
+
+type indexReportFixure struct {
+	IndexReport *claircore.IndexReport
+}
+
+type indexReportCtx struct {
+	CurLayer         claircore.Digest
+	CurDistribution  *claircore.Distribution
+	CurSource        *claircore.Package
+	CurPackageDB     string
+	CurRepositoryIDs []string
+
+	ManifestSet bool
+	LayerSet    bool
+}
+
+func (f *indexReportFixure) Value() *claircore.IndexReport {
+	return f.IndexReport
+}
+
+func (f *indexReportFixure) commonChecks(pc *indexReportCtx, args []string) error {
+	switch {
+	case len(args) == 0:
+		return errors.New("bad number of arguments: want 1 or more")
+	case !pc.ManifestSet:
+		return errors.New("bad command: no Manifest created")
+	case !pc.LayerSet:
+		return errors.New("bad command: no Layer created")
+	}
+	return nil
+}
+
+func (f *indexReportFixure) AddManifest(pc *indexReportCtx, args []string) (err error) {
+	if len(args) != 1 {
+		return errors.New("bad number of arguments: want exactly 1")
+	}
+	if pc.ManifestSet {
+		return errors.New("bad command: Manifest already created")
+	}
+	f.IndexReport.Hash, err = claircore.ParseDigest(args[0])
+	if err != nil {
+		return err
+	}
+	pc.ManifestSet = true
+	return nil
+}
+
+func (f *indexReportFixure) AddLayer(pc *indexReportCtx, args []string) (err error) {
+	if len(args) != 1 {
+		return errors.New("bad number of arguments: want exactly 1")
+	}
+	if !pc.ManifestSet {
+		return errors.New("bad command: no Manifest created")
+	}
+	pc.CurLayer, err = claircore.ParseDigest(args[0])
+	return err
+}
+
+func (f *indexReportFixure) AddDistribution(pc *indexReportCtx, args []string) error {
+	f.commonChecks(pc, args)
+	d := claircore.Distribution{}
+	if err := AssignToStruct(&d, args); err != nil {
+		return err
+	}
+	pc.CurDistribution = &d
+	return nil
+}
+
+func (f *indexReportFixure) ClearDistribution(pc *indexReportCtx, args []string) error {
+	if len(args) == 0 {
+		return errors.New("bad number of arguments: want 0")
+	}
+	pc.CurDistribution = nil
+	return nil
+}
+
+func (f *indexReportFixure) PushRepository(pc *indexReportCtx, args []string) error {
+	f.commonChecks(pc, args)
+	r := claircore.Repository{}
+	if err := AssignToStruct(&r, args); err != nil {
+		return err
+	}
+	if r.ID == "" {
+		r.ID = strconv.FormatInt(int64(len(f.IndexReport.Repositories)), 10)
+	}
+	f.IndexReport.Repositories[r.ID] = &r
+	pc.CurRepositoryIDs = append(pc.CurRepositoryIDs, r.ID)
+	return nil
+}
+
+func (f *indexReportFixure) PopRepository(pc *indexReportCtx, args []string) error {
+	if len(args) != 0 {
+		return errors.New("bad number of arguments: want 0")
+	}
+	last := len(pc.CurRepositoryIDs) - 1
+	pc.CurRepositoryIDs = pc.CurRepositoryIDs[:last:last] // Forces a unique slice when down-sizing.
+	return nil
+}
+
+func (f *indexReportFixure) AddPackage(pc *indexReportCtx, args []string) error {
+	f.commonChecks(pc, args)
+	p := claircore.Package{}
+	if err := AssignToStruct(&p, args); err != nil {
+		return err
+	}
+	if p.ID == "" {
+		p.ID = strconv.FormatInt(int64(len(f.IndexReport.Packages)), 10)
+	}
+	p.Source = pc.CurSource
+	f.IndexReport.Packages[p.ID] = &p
+	env := claircore.Environment{
+		PackageDB:     p.PackageDB,
+		IntroducedIn:  pc.CurLayer,
+		RepositoryIDs: pc.CurRepositoryIDs,
+	}
+	if pc.CurDistribution != nil {
+		env.DistributionID = pc.CurDistribution.ID
+	}
+	f.IndexReport.Environments[p.ID] = []*claircore.Environment{&env}
+	return nil
+}

test/fixturescript/indexreport_test.go

@@ -0,0 +1,49 @@
+package fixturescript
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+func ExampleCreateIndexReport() {
+	const example = `# Sample IndexReport Fixture
+Manifest sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Layer    sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+Repository URI=http://example.com/os-repo
+Package Name=hello Version=2.12 PackageDB=bdb:var/lib/rpm
+Layer    sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
+PopRepository
+Repository URI=http://example.com/my-repo
+Package Name=bash Version=5.2.26 PackageDB=bdb:var/lib/rpm
+`
+	report, err := CreateIndexReport("script", strings.NewReader(example))
+	if err != nil {
+		panic(err)
+	}
+	pkgIDs := make([]string, 0, len(report.Packages))
+	for id := range report.Packages {
+		pkgIDs = append(pkgIDs, id)
+	}
+	sort.Strings(pkgIDs)
+	fmt.Println("Manifest:", report.Hash)
+	for _, id := range pkgIDs {
+		pkg := report.Packages[id]
+		fmt.Println("Package:", pkg.Name, pkg.Version)
+		for _, env := range report.Environments[id] {
+			fmt.Println("\tLayer:", env.IntroducedIn)
+			fmt.Println("\tPackage DB:", env.PackageDB)
+			fmt.Println("\tRepositories:", env.RepositoryIDs)
+		}
+	}
+	// Output:
+	// Manifest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+	// Package: hello 2.12
+	// 	Layer: sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+	// 	Package DB: bdb:var/lib/rpm
+	// 	Repositories: [0]
+	// Package: bash 5.2.26
+	// 	Layer: sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
+	// 	Package DB: bdb:var/lib/rpm
+	// 	Repositories: [1]
+}