🫁 Transplant registry (#12)

Transplant registry from jig. Use protosync to download google api files.

registry.Files is wrapper for the official protoregistry.Files,
implementing protoregistry.MessageTypeResolver and
protoregistry.ExtensionTypeResolver, so that we can easily use this Files
wrapper with prototest and protojson to unmarshal extension.

A registry is created from a FileDescriptorSet like so:

    var fds *descriptorpb.FileDescriptorSet              // initialise
    protoregistryFiles, err := protodesc.NewFiles(&fds)  // handle err
    files := registry.NewFiles(protoregistryFiles)

Hermitise tools in preparation and to "vendorise" google/api and
google/protobuf with protosync.

This merges the following commits:
* Hermitise tools
* Add debug info on up-to-date checks
* Transplant registry
* Refactor files.go

     Makefile                                      |  14 +-
     ...rotoc-3.17.3.pkg => .gosimports-0.1.5.pkg} |   0
     bin/.protoc-3.19.4.pkg                        |   1 +
     bin/.protoc-gen-go-1.27.1.pkg                 |   1 +
     bin/.protoc-gen-go-grpc-1.1.0.pkg             |   1 +
     bin/.protosync-0.2.1.pkg                      |   1 +
     bin/.reflect-0.0.22.pkg                       |   1 +
     bin/gosimports                                |   1 +
     bin/hermit-packages/reflect.hcl               |   7 +
     bin/hermit.hcl                                |   1 +
     bin/protoc                                    |   2 +-
     bin/protoc-gen-go                             |   1 +
     bin/protoc-gen-go-grpc                        |   1 +
     bin/protosync                                 |   1 +
     bin/reflect                                   |   1 +
     httprule/internal/echo.pb.go                  |   2 +-
     httprule/internal/test.pb.go                  |   2 +-
     registry/files.go                             | 118 +++
     registry/files_test.go                        | 168 ++++
     .../testdata/google/api/annotations.proto     |  31 +
     registry/testdata/google/api/http.proto       | 375 +++++++
     .../testdata/google/protobuf/descriptor.proto | 921 ++++++++++++++++++
     registry/testdata/regtest.pb                  | Bin 0 -> 9425 bytes
     registry/testdata/regtest.proto               |  51 +
     24 files changed, 1690 insertions(+), 12 deletions(-)

Pull-Request: #12

Makefile

@@ -44,11 +44,13 @@ gen-pb = protoc -o $(1:%.proto=%-protoc.pb) $(1)
 gen-json = reflect fdsf $(1:%.proto=%-protoc.pb) -f json | jq . > $(1:%.proto=%-protoc.json)
 gen-testdata = $(call gen-pb,$(1))$(nl)$(call gen-json,$(1))$(nl)
 
-gen-testdata: tools
+gen-testdata:
 	$(foreach proto,$(wildcard testdata/*.proto),$(call gen-testdata,$(proto)))
+	protosync --dest registry/testdata google/api/annotations.proto
+	protoc --include_imports -I registry/testdata -o registry/testdata/regtest.pb registry/testdata/regtest.proto
 
 check-uptodate: gen-testdata protos
-	test -z "$$(git status --porcelain)"
+	test -z "$$(git status --porcelain)" || { git diff; false; }
 
 CHECK_COVERAGE = awk -F '[ \t%]+' '/^total:/ {print; if ($$3 < $(COVERAGE)) exit 1}'
 FAIL_COVERAGE = { echo '$(COLOUR_RED)FAIL - Coverage below $(COVERAGE)%$(COLOUR_NORMAL)'; exit 1; }
@@ -70,7 +72,7 @@ protos:
 		--go-grpc_out=. --go-grpc_opt=module=foxygo.at/protog \
 		-I httprule/internal \
 		test.proto echo.proto
-	goimports -w .
+	gosimports -w .
 
 .PHONY: protos
 
@@ -93,12 +95,6 @@ COLOUR_WHITE  = $(shell tput setaf 7 2>/dev/null)
 help:
 	@awk -F ':.*## ' 'NF == 2 && $$1 ~ /^[A-Za-z0-9%_-]+$$/ { printf "$(COLOUR_WHITE)%-25s$(COLOUR_NORMAL)%s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort
 
-tools:
-	go install google.golang.org/protobuf/cmd/[email protected]
-	go install google.golang.org/grpc/cmd/[email protected]
-	go install golang.org/x/tools/cmd/[email protected]
-	go install github.com/juliaogris/[email protected]
-
 $(O):
 	@mkdir -p $@
 

bin/.protoc-3.19.4.pkg

@@ -0,0 +1 @@
+hermit

bin/.protoc-gen-go-1.27.1.pkg

@@ -0,0 +1 @@
+hermit

bin/.protoc-gen-go-grpc-1.1.0.pkg

@@ -0,0 +1 @@
+hermit

bin/.protosync-0.2.1.pkg

@@ -0,0 +1 @@
+hermit

bin/.reflect-0.0.22.pkg

@@ -0,0 +1 @@
+hermit

bin/gosimports

@@ -0,0 +1 @@
+.gosimports-0.1.5.pkg

bin/hermit-packages/reflect.hcl

@@ -0,0 +1,7 @@
+description = "reflect"
+test        = "reflect --version"
+binaries    = ["reflect"]
+
+version "0.0.22" {
+  source = "https://github.com/juliaogris/reflect/releases/download/v${version}/reflect_${version}_${os}_${arch}.tar.gz"
+}

bin/hermit.hcl

@@ -1,3 +1,4 @@
+sources = ["env:///bin/hermit-packages", "https://github.com/cashapp/hermit-packages.git"]
 manage-git = false
 env = {
   GOBIN : "${HERMIT_ENV}/out/bin",

bin/protoc

@@ -1 +1 @@
-.protoc-3.17.3.pkg
+.protoc-3.19.4.pkg

bin/protoc-gen-go

@@ -0,0 +1 @@
+.protoc-gen-go-1.27.1.pkg

bin/protoc-gen-go-grpc

@@ -0,0 +1 @@
+.protoc-gen-go-grpc-1.1.0.pkg

bin/protosync

@@ -0,0 +1 @@
+.protosync-0.2.1.pkg

bin/reflect

@@ -0,0 +1 @@
+.reflect-0.0.22.pkg

registry/files.go

@@ -0,0 +1,118 @@
+// Package registry provides a type on top of protoregistry.Files that can be
+// used as a protoregistry.ExtensionTypeResolver and a
+// protoregistry.MessageTypeResolver. This allows a protoregistry.Files to be
+// used as Resolver for protobuf encoding marshaling options.
+package registry
+
+import (
+	"strings"
+
+	"google.golang.org/protobuf/reflect/protodesc"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/reflect/protoregistry"
+	"google.golang.org/protobuf/types/descriptorpb"
+	"google.golang.org/protobuf/types/dynamicpb"
+)
+
+type Files struct {
+	protoregistry.Files
+}
+
+func NewFiles(fds *descriptorpb.FileDescriptorSet) (*Files, error) {
+	f, err := protodesc.NewFiles(fds)
+	if err != nil {
+		return nil, err
+	}
+	return &Files{Files: *f}, nil
+}
+
+type extMatchFn func(protoreflect.ExtensionDescriptor) bool
+
+// extensionContainer is implemented by FileDescriptor and MessageDescriptor.
+// They are both "namespaces" that contain extensions and have "sub-namespaces".
+type extensionContainer interface {
+	Messages() protoreflect.MessageDescriptors
+	Extensions() protoreflect.ExtensionDescriptors
+}
+
+func (f *Files) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) {
+	desc, err := f.FindDescriptorByName(field)
+	if err != nil {
+		return nil, err
+	}
+	ed, ok := desc.(protoreflect.ExtensionDescriptor)
+	if !ok {
+		return nil, protoregistry.NotFound
+	}
+	return dynamicpb.NewExtensionType(ed), nil
+}
+
+func (f *Files) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) {
+	ets := f.walkExtensions(false, func(ed protoreflect.ExtensionDescriptor) bool {
+		return ed.ContainingMessage().FullName() == message && ed.Number() == field
+	})
+	if len(ets) == 0 {
+		return nil, protoregistry.NotFound
+	}
+	return ets[0], nil
+}
+
+func (f *Files) GetExtensionsOfMessage(message protoreflect.FullName) []protoreflect.ExtensionType {
+	return f.walkExtensions(true, func(ed protoreflect.ExtensionDescriptor) bool {
+		return ed.ContainingMessage().FullName() == message
+	})
+}
+
+func (f *Files) walkExtensions(getAll bool, pred extMatchFn) []protoreflect.ExtensionType {
+	var result []protoreflect.ExtensionType
+
+	f.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
+		result = append(result, getExtensions(fd, getAll, pred)...)
+		// continue if we are getting all extensions or have none so far
+		return getAll || len(result) == 0
+	})
+	return result
+}
+
+func getExtensions(ec extensionContainer, getAll bool, pred extMatchFn) []protoreflect.ExtensionType {
+	var result []protoreflect.ExtensionType
+
+	eds := ec.Extensions()
+	for i := 0; i < eds.Len() && (getAll || len(result) == 0); i++ {
+		ed := eds.Get(i)
+		if pred(ed) {
+			result = append(result, dynamicpb.NewExtensionType(ed))
+		}
+	}
+
+	mds := ec.Messages()
+	for i := 0; i < mds.Len() && (getAll || len(result) == 0); i++ {
+		md := mds.Get(i)
+		result = append(result, getExtensions(md, getAll, pred)...)
+	}
+
+	return result
+}
+
+func (f *Files) FindMessageByName(name protoreflect.FullName) (protoreflect.MessageType, error) {
+	desc, err := f.FindDescriptorByName(name)
+	if err != nil {
+		return nil, err
+	}
+	md, ok := desc.(protoreflect.MessageDescriptor)
+	if !ok {
+		return nil, protoregistry.NotFound
+	}
+	return dynamicpb.NewMessageType(md), nil
+}
+
+func (f *Files) FindMessageByURL(url string) (protoreflect.MessageType, error) {
+	message := protoreflect.FullName(url)
+	// Strip off before the last slash - we only look locally for the
+	// message and do not hit the network. The part after the last slash
+	// must be the full name of the message.
+	if i := strings.LastIndexByte(url, '/'); i >= 0 {
+		message = message[i+len("/"):]
+	}
+	return f.FindMessageByName(message)
+}

registry/files_test.go

@@ -0,0 +1,168 @@
+package registry
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/reflect/protoregistry"
+	"google.golang.org/protobuf/types/descriptorpb"
+)
+
+// ensure Files implments ExtensionTypeResolver
+var _ protoregistry.ExtensionTypeResolver = (*Files)(nil)
+
+// ensure Files implments MessageTypeResolver
+var _ protoregistry.MessageTypeResolver = (*Files)(nil)
+
+func TestFindExtensionByName(t *testing.T) {
+	tests := map[string]struct {
+		extName string
+		err     error
+	}{
+		"top-level extension":      {"regtest.ef1", nil},
+		"nested extension":         {"regtest.ExtensionMessage.ef2", nil},
+		"deeply nested extension":  {"regtest.ExtensionMessage.NestedExtension.ef3", nil},
+		"other package extension":  {"regtest.base", nil},
+		"imported extension":       {"google.api.http", nil},
+		"unknown extension":        {"unknown.extension", protoregistry.NotFound},
+		"non-extension descriptor": {"regtest.BaseMessage", protoregistry.NotFound},
+	}
+
+	f := newFiles(t)
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			extName := protoreflect.FullName(tc.extName)
+			et, err := f.FindExtensionByName(extName)
+			if tc.err != nil {
+				require.ErrorIs(t, err, tc.err)
+			} else {
+				require.NoError(t, err, tc.extName)
+				require.Equal(t, extName, et.TypeDescriptor().FullName())
+			}
+		})
+	}
+}
+
+func TestFindExtensionByNumber(t *testing.T) {
+	tests := map[string]struct {
+		message     string
+		fieldNumber int32
+		extName     string
+		err         error
+	}{
+		"top-level extension":     {"regtest.BaseMessage", 1000, "regtest.ef1", nil},
+		"nested extension":        {"regtest.BaseMessage", 1001, "regtest.ExtensionMessage.ef2", nil},
+		"deeply nested extension": {"regtest.BaseMessage", 1002, "regtest.ExtensionMessage.NestedExtension.ef3", nil},
+		"other package extension": {"google.protobuf.MethodOptions", 56789, "regtest.base", nil},
+		"imported extension":      {"google.protobuf.MethodOptions", 72295728, "google.api.http", nil},
+		"unknown message":         {"regtest.Foo", 999, "unknown.message", protoregistry.NotFound},
+		"unknown extension":       {"regtest.BaseMessage", 999, "unknown.extension", protoregistry.NotFound},
+	}
+
+	f := newFiles(t)
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			messageName := protoreflect.FullName(tc.message)
+			fieldNumber := protoreflect.FieldNumber(tc.fieldNumber)
+			et, err := f.FindExtensionByNumber(messageName, fieldNumber)
+			if tc.err != nil {
+				require.ErrorIs(t, err, tc.err)
+			} else {
+				require.NoError(t, err, tc.extName)
+				extName := protoreflect.FullName(tc.extName)
+				require.Equal(t, extName, et.TypeDescriptor().FullName())
+			}
+		})
+	}
+}
+
+func TestGetExtensionsOfMessage(t *testing.T) {
+	tests := map[string]struct {
+		message string
+		fields  []int32
+	}{
+		"package message":  {"regtest.BaseMessage", []int32{1000, 1001, 1002}},
+		"imported message": {"google.protobuf.MethodOptions", []int32{56789, 72295728}},
+		"unknown message":  {"regtest.Foo", nil},
+	}
+
+	f := newFiles(t)
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			messageName := protoreflect.FullName(tc.message)
+			ets := f.GetExtensionsOfMessage(messageName)
+			var fields []int32
+			for _, et := range ets {
+				fields = append(fields, int32(et.TypeDescriptor().Number()))
+			}
+			require.ElementsMatch(t, tc.fields, fields)
+		})
+	}
+}
+
+func TestFindMessageByName(t *testing.T) {
+	tests := map[string]struct {
+		name string
+		err  error
+	}{
+		"top-level message":      {"regtest.BaseMessage", nil},
+		"nested message":         {"regtest.ExtensionMessage.NestedExtension", nil},
+		"unknown message":        {"regtest.Foo", protoregistry.NotFound},
+		"non-message descriptor": {"regtest.ef1", protoregistry.NotFound},
+	}
+
+	f := newFiles(t)
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			messageName := protoreflect.FullName(tc.name)
+			mt, err := f.FindMessageByName(messageName)
+			if tc.err != nil {
+				require.ErrorIs(t, err, tc.err)
+			} else {
+				require.NoError(t, err, tc.name)
+				require.Equal(t, messageName, mt.Descriptor().FullName())
+			}
+		})
+	}
+}
+
+func TestFindMessageByURL(t *testing.T) {
+	tests := map[string]struct {
+		url string
+		err error
+	}{
+		"simple url":       {"regtest.BaseMessage", nil},
+		"hostname url":     {"example.com/regtest.BaseMessage", nil},
+		"multiple slashes": {"example.com/foo/bar/regtest.BaseMessage", nil},
+		"unknown message":  {"example.com/regtest.Foo", protoregistry.NotFound},
+	}
+
+	f := newFiles(t)
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			mt, err := f.FindMessageByURL(tc.url)
+			if tc.err != nil {
+				require.ErrorIs(t, err, tc.err)
+			} else {
+				require.NoError(t, err, tc.url)
+				expected := protoreflect.FullName("regtest.BaseMessage")
+				require.Equal(t, expected, mt.Descriptor().FullName())
+			}
+		})
+	}
+}
+
+func newFiles(t *testing.T) *Files {
+	t.Helper()
+	b, err := os.ReadFile("testdata/regtest.pb")
+	require.NoError(t, err)
+	fds := descriptorpb.FileDescriptorSet{}
+	err = proto.Unmarshal(b, &fds)
+	require.NoError(t, err)
+	files, err := NewFiles(&fds)
+	require.NoError(t, err)
+	return files
+}