feat: add extism-dev update --wasm to update all the test wasm files across our repos

extism-dev/args.go

@@ -94,6 +94,19 @@ func (data *extismData) mergeRepos() {
 	})
 }
 
+func (data *extismData) mergeExtraWasm() {
+	m := map[string]string{}
+	for k, v := range defaultTestWasm {
+		m[k] = v
+	}
+
+	for k, v := range data.TestWasm {
+		m[k] = v
+	}
+
+	data.TestWasm = m
+}
+
 func (a *devArgs) SetArgs(args []string) {
 	a.args = args
 }

extism-dev/dev.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	_ "embed"
 	"encoding/json"
 	"os"
 	"path/filepath"
@@ -9,8 +10,28 @@ import (
 	"github.com/spf13/cobra"
 )
 
+//go:embed repos.json
+var repos []byte
+
+//go:embed testWasm.json
+var testWasm []byte
+
+var defaultRepos []repo
+var defaultTestWasm map[string]string
+
 type extismData struct {
-	Repos []repo `json:"repos"`
+	Repos    []repo            `json:"repos"`
+	TestWasm map[string]string `json:"testWasm,omitempty"`
+}
+
+func init() {
+	if err := json.Unmarshal(repos, &defaultRepos); err != nil {
+		panic(err)
+	}
+
+	if err := json.Unmarshal(testWasm, &defaultTestWasm); err != nil {
+		panic(err)
+	}
 }
 
 func homeDir() string {
@@ -107,6 +128,7 @@ func SetupDevCmd(dev *cobra.Command) error {
 	devFind.Flags().StringVar(&findArgs.editor, "editor", defaultEditor, "Editor command")
 	devFind.Flags().BoolVar(&findArgs.edit, "edit", false, "Edit matching files")
 	devFind.Flags().BoolVarP(&findArgs.interactive, "interactive", "i", false, "Prompt before editing or replacing")
+	devFind.Flags().BoolVar(&findArgs.dryRun, "dry-run", false, "Don't actually write any files when --replace or --edit are passed")
 	dev.AddCommand(devFind)
 
 	// Add
@@ -215,7 +237,12 @@ func SetupDevCmd(dev *cobra.Command) error {
 		RunE:         cli.RunArgs(runDevUpdate, updateArgs),
 	}
 	devUpdate.Flags().BoolVar(&updateArgs.kernel, "kernel", false, "Update kernel files across repos")
+	devUpdate.Flags().BoolVar(&updateArgs.wasm, "wasm", false, "Update test wasm files across repos")
 	devUpdate.Flags().BoolVar(&updateArgs.all, "all", false, "Enable all updates")
+	devUpdate.Flags().BoolVar(&updateArgs.dryRun, "dry-run", false, "Don't actually write any files")
+	devUpdate.Flags().BoolVarP(&updateArgs.build, "build", "b", false, "Run any necesarry build steps first")
+	devUpdate.Flags().StringVarP(&updateArgs.repo, "repo", "r", "", "Regex filter used on the repo name")
+	devUpdate.Flags().StringVarP(&updateArgs.category, "category", "c", "", "Category: sdk, pdk, plugin, runtime or other")
 	dev.AddCommand(devUpdate)
 
 	return nil

extism-dev/find.go

@@ -19,6 +19,7 @@ type devFindArgs struct {
 	editor      string
 	repo        string
 	replace     string
+	dryRun      bool
 	interactive bool
 }
 
@@ -84,6 +85,10 @@ func runDevFind(cmd *cobra.Command, args *devFindArgs) error {
 			return search.Iter(func(path string) error {
 				editLock.Lock()
 				defer editLock.Unlock()
+				if args.dryRun {
+					cli.Print("Edit", path)
+					return nil
+				}
 				if !args.prompt("Edit ", path) {
 					return nil
 				}

extism-dev/go.mod

@@ -2,10 +2,10 @@ module github.com/extism/cli/extism-dev
 
 go 1.20
 
-replace github.com/extism/cli => ../
+// replace github.com/extism/cli => ../
 
 require (
-	github.com/extism/cli v0.3.0
+	github.com/extism/cli v0.3.1-0.20231014144908-9e2d6c2cfa56
 	github.com/extism/go-sdk v0.0.0-20231013005816-307a3b3634a4
 	github.com/gobwas/glob v0.2.3
 	github.com/spf13/cobra v1.7.0

extism-dev/go.sum

@@ -5,24 +5,32 @@ github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtM
 github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/ebitengine/purego v0.4.0 h1:RQVuMIxQPQ5iCGEJvjQ17YOK+1tMKjVau2FUMvXH4HE=
 github.com/ebitengine/purego v0.4.0/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
+github.com/extism/cli v0.3.0 h1:o25toFiLNe+x5VvTeCdmdSc9wRzzeHyL3ATL8xEGVeY=
+github.com/extism/cli v0.3.0/go.mod h1:OE8YOuZxnV3g8SR6R3KNP9ziSH+2xJwBXuf0xLXxLOA=
+github.com/extism/cli v0.3.1-0.20231014144908-9e2d6c2cfa56 h1:5FtzgTLpe9tfhhMjqaR45bCVnCccd/0TdUWMKDSzmAg=
+github.com/extism/cli v0.3.1-0.20231014144908-9e2d6c2cfa56/go.mod h1:/ntVkZPTbNh/SjkO5OkTlDmaIt5/UOIy/yKjPcEpM9Y=
 github.com/extism/go-sdk v0.0.0-20231013005816-307a3b3634a4 h1:j8qtkXS9qaVXnyZsLA6exXsJIBtr5WlBFrfVSC/JRnA=
 github.com/extism/go-sdk v0.0.0-20231013005816-307a3b3634a4/go.mod h1:xUfKSEQndAvHBc1Ohdre0e+UdnRzUpVfbA8QLcx4fbY=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
 github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
 github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/tetratelabs/wazero v1.3.0 h1:nqw7zCldxE06B8zSZAY0ACrR9OH5QCcPwYmYlwtcwtE=
 github.com/tetratelabs/wazero v1.3.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -39,4 +47,5 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

extism-dev/init.go

@@ -26,6 +26,7 @@ func runDevInit(cmd *cobra.Command, args *devInitArgs) error {
 		}
 	} else {
 		data.mergeRepos()
+		data.mergeExtraWasm()
 	}
 
 	if args.category != "" {

extism-dev/repo.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	_ "embed"
-	"encoding/json"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -11,9 +9,6 @@ import (
 	"github.com/extism/cli"
 )
 
-//go:embed repos.json
-var repos []byte
-
 type repo struct {
 	Url      string `json:"url"`
 	Category string `json:"category"`
@@ -55,11 +50,3 @@ func (repo repo) clone() bool {
 
 	return false
 }
-
-var defaultRepos []repo
-
-func init() {
-	if err := json.Unmarshal(repos, &defaultRepos); err != nil {
-		panic(err)
-	}
-}

extism-dev/search.go

@@ -121,8 +121,12 @@ func (search *Search) Replace(r string) error {
 					return nil
 				}
 				cli.Print("Updating", abs)
-				data := search.rx.ReplaceAll(data, []byte(r))
-				return ioutil.WriteFile(abs, data, entry.Mode().Perm())
+				if search.args.dryRun {
+					return nil
+				} else {
+					data := search.rx.ReplaceAll(data, []byte(r))
+					return ioutil.WriteFile(abs, data, entry.Mode().Perm())
+				}
 			}
 
 			return nil

extism-dev/testWasm.json

@@ -0,0 +1,4 @@
+{
+  "globals.wasm": "extism/c-pdk/examples/globals/globals.wasm",
+  "code-functions.wasm": "extism/c-pdk/examples/host-functions/host-functions.wasm"
+}

extism-dev/update.go

@@ -2,16 +2,49 @@ package main
 
 import (
 	"io/ioutil"
+	"os"
+	"os/exec"
 	"path/filepath"
+	"regexp"
 
 	"github.com/extism/cli"
 	"github.com/spf13/cobra"
 )
 
 type devUpdateArgs struct {
 	devArgs
-	kernel bool
-	all    bool
+	kernel   bool
+	wasm     bool
+	all      bool
+	dryRun   bool
+	repo     string
+	category string
+	build    bool
+}
+
+type wasmSource struct {
+	path string
+	mode os.FileMode
+	data []byte
+}
+
+func (w *wasmSource) Get() ([]byte, error) {
+	if len(w.data) > 0 {
+		return w.data, nil
+	}
+	s, err := os.Stat(w.path)
+	if err != nil {
+		return []byte{}, err
+	}
+	w.mode = s.Mode()
+
+	d, err := ioutil.ReadFile(w.path)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	w.data = d
+	return w.data, nil
 }
 
 func runDevUpdate(cmd *cobra.Command, args *devUpdateArgs) error {
@@ -21,28 +54,130 @@ func runDevUpdate(cmd *cobra.Command, args *devUpdateArgs) error {
 	}
 
 	kernelPath := args.Path("extism", "extism", "runtime", "src", "extism-runtime.wasm")
-	kernelData, err := ioutil.ReadFile(kernelPath)
-	if err != nil {
-		return err
+	kernel := wasmSource{path: kernelPath}
+
+	if args.all || args.kernel {
+		cmd := exec.Command("bash", "build.sh")
+		cmd.Dir = args.Path("extism", "extism", "kernel")
+		cli.Print("Building extism-runtime.wasm")
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			cli.Log("Error building kernel:", err)
+			cli.Print(string(output))
+			return err
+		}
+	}
+
+	if args.all || args.wasm {
+		cmd := exec.Command("make")
+		cmd.Dir = args.Path("extism", "plugins")
+		cli.Print("Building plugins in extism/plugins")
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			cli.Log("Error building plugins:", err)
+			cli.Print(string(output))
+			return err
+		}
+
+		cmd = exec.Command("make")
+		cmd.Dir = args.Path("extism", "c-pdk")
+		cli.Print("Building plugins in extism/c-pdk")
+		output, err = cmd.CombinedOutput()
+		if err != nil {
+			cli.Log("Error building c-pdk plugins:", err)
+			cli.Print(string(output))
+			return err
+		}
 	}
 
 	repos := []repo{}
 	for _, repo := range data.Repos {
+		if args.category != "" && repo.Category != args.category {
+			continue
+		}
+
+		if args.repo != "" {
+			rx := regexp.MustCompile(args.repo)
+			if !rx.Match([]byte(repo.path())) {
+				continue
+			}
+		}
 		repos = append(repos, repo)
 	}
+
+	// Get configured extra wasm files
+	dataFile, err := args.loadDataFile()
+	if err != nil {
+		cli.Log("Unable to load data file", err)
+		return err
+	}
+	sources := map[string]wasmSource{}
+
+	for k, v := range dataFile.TestWasm {
+		sources[k] = wasmSource{path: args.Path(v)}
+	}
+
+	pluginsDir := args.Path("extism", "plugins", "target", "wasm32-unknown-unknown", "release")
+
+	// Get plugins from `plugins` directory
+	files, err := os.ReadDir(pluginsDir)
+	if err != nil {
+		files = []os.DirEntry{}
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		name := file.Name()
+		sources[name] = wasmSource{path: filepath.Join(pluginsDir, name)}
+		if name == "count_vowels.wasm" {
+			sources["code.wasm"] = sources[name]
+		} else if name == "loop_forever.wasm" {
+			sources["loop.wasm"] = sources[name]
+		}
+	}
+
 	search := NewSearch(nil, "", repos...)
 	search.Iter(func(name string) error {
 		fname := filepath.Base(name)
 
+		// Update kernel
 		if args.all || args.kernel {
 			if fname == "extism-runtime.wasm" && name != kernelPath {
 				cli.Print("Updating", name)
-				err := ioutil.WriteFile(name, kernelData, 0o655)
+				if !args.dryRun {
+					k, err := kernel.Get()
+					if err != nil {
+						cli.Log("Unable to load kernel", err)
+						return err
+					}
+					err = ioutil.WriteFile(name, k, 0o655)
+					if err != nil {
+						cli.Print("Error copying extism-kernel file to", name+":", err)
+					}
+				}
+			}
+		}
+
+		// Update test wasm modules
+		if args.all || args.wasm {
+			if p, ok := sources[fname]; ok && name != p.path {
+				cli.Print("Updating", name)
+				data, err := p.Get()
 				if err != nil {
-					cli.Print("Error copying extism-kernel file", err)
+					cli.Print("Error reading file", err)
+					return err
+				}
+				if !args.dryRun {
+					err = ioutil.WriteFile(name, data, p.mode)
+					if err != nil {
+						cli.Print("Error copying", p, "to", name+":", err)
+					}
 				}
 			}
 		}
+
 		return nil
 	})