pb: Initialise pb cli

Create basic `pb` command converting proto message from one encoding
format to another, currently supporting JSON, prototext, binary proto.

Add basic docs, started by @camh- and goreleaser config.

Signed-off-by: Julia Ogris <[email protected]>

.gitattributes

@@ -0,0 +1,2 @@
+*.pb   linguist-generated=true
+go.sum linguist-generated=true

.goreleaser.yml

@@ -0,0 +1,11 @@
+dist: out/dist
+builds:
+  - main: ./cmd/pb
+    id: pb
+    binary: pb
+archives:
+  - builds: ['pb']
+    id: all
+  - builds: ['pb']
+    id: pb
+    name_template: 'pb_{{.Version}}_{{.Os}}_{{.Arch}}'

Makefile

@@ -2,7 +2,7 @@
 
 # --- Global -------------------------------------------------------------------
 O = out
-COVERAGE = 90
+COVERAGE = 85
 VERSION ?= $(shell git describe --tags --dirty  --always)
 REPO_ROOT = $(shell git rev-parse --show-toplevel)
 
@@ -18,13 +18,13 @@ clean::  ## Remove generated files
 .PHONY: all ci clean
 
 # --- Build --------------------------------------------------------------------
-GO_LDFLAGS = -X main.version=$(VERSION)
+CMDS = ./cmd/pb
 
 build: | $(O)  ## Build reflect binaries
-	go build -o $(O)/protog -ldflags='$(GO_LDFLAGS)' .
+	go build -o $(O) $(CMDS)
 
 install:  ## Build and install binaries in $GOBIN
-	go install -ldflags='$(GO_LDFLAGS)' .
+	go install $(CMDS)
 
 .PHONY: build install
 
@@ -50,16 +50,15 @@ FAIL_COVERAGE = { echo '$(COLOUR_RED)FAIL - Coverage below $(COVERAGE)%$(COLOUR_
 .PHONY: check-coverage check-uptodate cover test
 
 # --- Lint ---------------------------------------------------------------------
-
 lint:  ## Lint go source code
 	golangci-lint run
 
 .PHONY: lint
 
 # --- Protos ---------------------------------------------------------------------
-
 proto:
 	protosync --dest proto google/api/annotations.proto
+	protoc -I cmd/pb/testdata --include_imports -o cmd/pb/testdata/pbtest.pb cmd/pb/testdata/pbtest.proto
 	protoc -I proto -I registry/testdata --include_imports -o registry/testdata/regtest.pb registry/testdata/regtest.proto
 	protoc -I proto -I httprule/internal --go_out=. --go_opt=module=foxygo.at/protog --go-grpc_out=. --go-grpc_opt=module=foxygo.at/protog test.proto echo.proto
 	gosimports -w .

README.md

@@ -0,0 +1,32 @@
+# A toolkit for working with protobuf and gRPC in Go
+
+[![ci](https://github.com/foxygoat/protog/actions/workflows/cicd.yaml/badge.svg?branch=master)](https://github.com/foxygoat/protog/actions/workflows/cicd.yaml?branch=master)
+[![Godoc](https://img.shields.io/badge/godoc-ref-blue)](https://pkg.go.dev/foxygo.at/protog)
+[![Slack chat](https://img.shields.io/badge/slack-gophers-795679?logo=slack)](https://gophers.slack.com/messages/foxygoat)
+
+protog is a toolkit for Google's protobuf and gRPC projects. It contains a
+couple of Go packages and command line tools.
+
+It is developed against the protobuf-go v2 API (google.golang.org/protobuf)
+which simplifies a lot of reflection-based protobuf code compared to v1
+(github.com/golang/protobuf).
+
+Build, test and install with `make`. See further options with `make help`.
+
+## pb
+
+`pb` is a CLI tool for converting proto messages between different formats.
+
+Sample usage:
+
+    # create base-massage.pb binary encoded proto message file from input JSON
+    pb -P cmd/pb/testdata/pbtest.pb -o base-message.pb BaseMessage '{"f" : "some_field"}'
+
+    # convert binary encoded proto message from pb file back to JSON
+    pb -P cmd/pb/testdata/pbtest.pb -O json BaseMessage @base-message.pb
+
+Output:
+
+    {
+      "f": "some_field"
+    }

cmd/pb/main.go

@@ -0,0 +1,248 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	"foxygo.at/protog/registry"
+	"github.com/alecthomas/kong"
+	"golang.org/x/sys/unix"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/encoding/prototext"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/types/descriptorpb"
+	"google.golang.org/protobuf/types/dynamicpb"
+)
+
+var (
+	version     = "tip"
+	commit      = "HEAD"
+	date        = "now"
+	description = `
+pb translates encoded Protobuf message from one format to another
+`
+	cli struct {
+		Version  kong.VersionFlag `help:"Show version."`
+		PBConfig PBConfig         `embed:""`
+	}
+)
+
+type PBConfig struct {
+	Protoset    *registry.Files `short:"P" help:"Protoset of Message being translated" required:""`
+	Out         string          `short:"o" help:"Output file name"`
+	InFormat    string          `short:"I" help:"Input format (json, pb)" enum:"json,pb,j,p," default:""`
+	OutFormat   string          `short:"O" help:"Output format (j[son], p[b],t[xt])" enum:"json,pb,txt,j,p,t," default:""`
+	MessageType string          `arg:"" help:"Message type to be translated" required:""`
+	In          string          `arg:"" help:"Message value JSON encoded" optional:""`
+}
+
+func main() {
+	kctx := kong.Parse(&cli,
+		kong.Description(description),
+		kong.Vars{"version": fmt.Sprintf("%s (%s on %s)", version, commit, date)},
+		kong.TypeMapper(reflect.TypeOf(cli.PBConfig.Protoset), kong.MapperFunc(registryMapper)),
+	)
+	err := run(cli.PBConfig)
+	kctx.FatalIfErrorf(err)
+}
+
+type unmarshaler func([]byte, proto.Message) error
+type marshaler func(proto.Message) ([]byte, error)
+
+func run(cfg PBConfig) error {
+	md, err := lookupMessage(cfg.Protoset, cfg.MessageType)
+	if err != nil {
+		return err
+	}
+	in, err := cfg.readInput()
+	if err != nil {
+		return err
+	}
+	unmarshal, err := cfg.unmarshaler()
+	if err != nil {
+		return err
+	}
+	message := dynamicpb.NewMessage(md)
+	if err := unmarshal(in, message); err != nil {
+		return err
+	}
+	marshal, err := cfg.marshaler()
+	if err != nil {
+		return err
+	}
+	b, err := marshal(message)
+	if err != nil {
+		return err
+	}
+	return cfg.writeOutput(b)
+}
+
+func (c *PBConfig) readInput() ([]byte, error) {
+	if c.In == "" {
+		return io.ReadAll(os.Stdin)
+	}
+	if strings.HasPrefix(c.In, "@") {
+		return os.ReadFile(c.In[1:])
+	}
+	return []byte(c.In), nil
+}
+
+func (c *PBConfig) writeOutput(b []byte) error {
+	if c.Out == "" {
+		if getFormat("", c.OutFormat) == "pb" && isTTY() {
+			return fmt.Errorf("not writing binary to terminal. Use -O json or -O txt to output a textual format")
+		}
+		_, err := os.Stdout.Write(b)
+		return err
+	}
+	return os.WriteFile(c.Out, b, 0666)
+}
+
+func (c *PBConfig) unmarshaler() (unmarshaler, error) {
+	format := getFormat(c.In, c.InFormat)
+	switch format {
+	case "json":
+		o := protojson.UnmarshalOptions{Resolver: c.Protoset}
+		return o.Unmarshal, nil
+	case "pb":
+		o := proto.UnmarshalOptions{Resolver: c.Protoset}
+		return o.Unmarshal, nil
+	case "txt":
+		o := prototext.UnmarshalOptions{Resolver: c.Protoset}
+		return o.Unmarshal, nil
+	}
+	return nil, fmt.Errorf("unknown input format %s", format)
+}
+
+func (c *PBConfig) marshaler() (marshaler, error) {
+	format := getFormat("@"+c.Out, c.OutFormat)
+	switch format {
+	case "json":
+		o := protojson.MarshalOptions{Resolver: c.Protoset, Multiline: true}
+		return func(m proto.Message) ([]byte, error) {
+			b, err := o.Marshal(m)
+			if err != nil {
+				return nil, err
+			}
+			return append(b, byte('\n')), nil
+		}, nil
+	case "pb":
+		o := proto.MarshalOptions{}
+		return o.Marshal, nil
+	case "txt":
+		o := prototext.MarshalOptions{Resolver: c.Protoset, Multiline: true}
+		return o.Marshal, nil
+	}
+	return nil, fmt.Errorf("unknown output format %s", format)
+}
+
+func getFormat(contentOrFile string, format string) string {
+	if format != "" {
+		return canonicalFormat(format)
+	}
+	ext := filepath.Ext(contentOrFile)
+	// default to JSON for stdout, string input and files without extension
+	if contentOrFile == "@" || !strings.HasPrefix(contentOrFile, "@") || ext == "" {
+		return "json"
+	}
+	return canonicalFormat(ext[1:])
+}
+
+func canonicalFormat(format string) string {
+	switch format {
+	case "json", "j":
+		return "json"
+	case "pb", "p":
+		return "pb"
+	case "txt", "t", "prototext", "prototxt":
+		return "txt"
+	}
+	return format
+}
+
+func lookupMessage(reg *registry.Files, name string) (protoreflect.MessageDescriptor, error) {
+	var result []protoreflect.MessageDescriptor
+	reg.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
+		for i := 0; i < fd.Messages().Len(); i++ {
+			md := fd.Messages().Get(i)
+			mds, exactMatch := lookupMessageInMD(md, name)
+			if exactMatch {
+				result = mds
+				return false
+			}
+			result = append(result, mds...)
+		}
+		return true
+	})
+
+	if len(result) == 0 {
+		return nil, fmt.Errorf("message not found: %s", name)
+	}
+	if len(result) > 1 {
+		return nil, fmt.Errorf("ambiguous message name: %s", name)
+	}
+	return result[0], nil
+}
+
+func lookupMessageInMD(md protoreflect.MessageDescriptor, name string) (mds []protoreflect.MessageDescriptor, exactMatch bool) {
+	mdName := string(md.FullName())
+	if name == mdName || name == "."+mdName {
+		// If we have a full name match, we're done and will also
+		// ignore any other partial name matches.
+		return []protoreflect.MessageDescriptor{md}, true
+	}
+	mdLowerName := "." + strings.ToLower(mdName)
+	lowerName := strings.ToLower(name)
+	if lowerName == mdLowerName || strings.HasSuffix(mdLowerName, "."+lowerName) {
+		mds = append(mds, md)
+	}
+	subMessages := md.Messages()
+	for i := 0; i < subMessages.Len(); i++ {
+		md = subMessages.Get(i)
+		subMDs, exactMatch := lookupMessageInMD(md, name)
+		if exactMatch {
+			return subMDs, true
+		}
+		mds = append(mds, subMDs...)
+	}
+	return mds, false
+}
+
+func registryMapper(kctx *kong.DecodeContext, target reflect.Value) error {
+	reg, ok := target.Interface().(*registry.Files)
+	if !ok {
+		panic("target is not a *registry.Files")
+	}
+	var filename string
+	if err := kctx.Scan.PopValueInto("file", &filename); err != nil {
+		return err
+	}
+	files, err := registryFiles(filename)
+	if err != nil {
+		return err
+	}
+	*reg = *files
+	return nil
+}
+
+func registryFiles(filename string) (*registry.Files, error) {
+	b, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	fds := descriptorpb.FileDescriptorSet{}
+	if err := proto.Unmarshal(b, &fds); err != nil {
+		return nil, err
+	}
+	return registry.NewFiles(&fds)
+}
+
+func isTTY() bool {
+	_, err := unix.IoctlGetWinsize(int(os.Stdout.Fd()), unix.TIOCGWINSZ)
+	return err == nil
+}