Improve proto generation and cleanup old protos

.pre-commit-config.yaml

@@ -15,3 +15,4 @@ repos:
       name: proto_lint
       language: system
       entry: etc/proto_lint/proto_lint.py
+      pass_filenames: false

Makefile

@@ -14,22 +14,20 @@ docs:
 	pdoc --html --html-dir docs python_pachyderm
 
 docker-build-proto:
-	cd proto && \
-		docker build -t pachyderm_python_proto .
+	docker build -t pachyderm_python_proto proto
 
-src/python_pachyderm/proto: docker-build-proto
+src/python_pachyderm/proto/v2: docker-build-proto
 	@echo "Building with pachyderm core v$(PACHYDERM_VERSION)"
-	rm -rf src/python_pachyderm/proto
+	rm -rf src/python_pachyderm/proto/v2
 	cd proto/pachyderm && \
 		git fetch --all && \
 		git checkout v$(PACHYDERM_VERSION)
-	find ./proto/pachyderm/ -regex ".*\.proto" \
+	find ./proto/pachyderm/src -regex ".*\.proto" \
 	| grep -v 'internal' \
+	| grep -v 'server' \
 	| xargs tar cf - \
-	| docker run -i pachyderm_python_proto \
-	| tar xf -
-	mv src/python_pachyderm/src src/python_pachyderm/proto
-	find src/python_pachyderm/proto -type d -exec touch {}/__init__.py \;
+	| docker run -e VERSION=2 -i pachyderm_python_proto \
+	| tar -C src -xf -
 
 init:
 	git submodule update --init

proto/Dockerfile

@@ -1,6 +1,8 @@
-FROM python:3-onbuild
+FROM python:3.6-slim
 LABEL maintainer="[email protected]"
 
+RUN pip3 install grpcio==1.38.0 grpcio-tools==1.38.0
+
 ADD run /bin
 ENTRYPOINT ["/bin/run"]
 WORKDIR /work

proto/run

@@ -1,32 +1,33 @@
-#!/bin/sh -x 
+#!/bin/bash
+set -e
 
+# will extract from stdin and to ./proto/pachyderm/src
 tar xf /dev/stdin
 
-OUTDIR=src/python_pachyderm/
-mkdir -p $OUTDIR
+# VERSION matches the MajorVersion of pachyderm
+OUTDIR="python_pachyderm/proto/v${VERSION}"
 
-for i in $(find ./proto/pachyderm/src -name "*.proto"); do \
-    # Make sure to remove the gogoproto line, as that is go specific
-    sed -i s/import.*gogo.proto.*\;// $i
-    sed -i 's/\[.*gogoproto.*\]//' $i
-    # We need to do this in a pass before compilation because it's not guaranteed we
-    # traverse the proto files in order of dependence
-done
+mkdir -p ${OUTDIR}
+mv ./proto/pachyderm/src/* ${OUTDIR}
 
-# fix imports to be relative to src/
-for i in $(find ./proto/pachyderm/src -name "*.proto" | grep -v "proto/pachyderm/src/internal"); do \
-    perl -pi.bak -e 's/import "((?!google).*)"/import "src\/$1"/' $i
+# Make sure to remove the gogoproto line, as that is Go specific
+for i in $(find ${OUTDIR} -name "*.proto"); do
+    # remove the import
+    sed -i 's/import.*gogo.proto.*\;//' ${i}
+    # remove usages of gogoproto types
+    sed -i 's/\[.*gogoproto.*\]//' ${i}
 done
 
-# skip pkg and server because protos in there aren't part of the public API
-for i in $(find ./proto/pachyderm/src -name "*.proto" | grep -v "proto/pachyderm/src/internal" | grep -v 'proto/pachyderm/src/server'); do \
-    python3 -m grpc_tools.protoc -I./proto/pachyderm --python_out=$OUTDIR --grpc_python_out=$OUTDIR ${i}; \
+# fix imports to be relative to OUTDIR
+for i in $(find ${OUTDIR} -name "*.proto"); do
+    perl -pi -e "s/import \"((?!google).*)\"/import \"python_pachyderm\/proto\/v${VERSION}\/\$1\"/" ${i}
 done
 
-for i in $(find $OUTDIR -regex ".*pb2.*py"); do
-    # Fix the imports so they work once this is packaged by pypi
-    sed -i 's/from src\./from python_pachyderm\.proto\./' $i
-done
+# generate protobuf and gRPC code from proto files
+# should result in {}_pb2.py and {}_pb2_grpc.py per proto
+find ${OUTDIR} -name '*.proto' | xargs python3 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=.
+find ${OUTDIR} -name '*.proto' | xargs rm
 
+find ${OUTDIR} -type d -exec touch {}/__init__.py \;
 
-find $OUTDIR -regex ".*pb2.*py" | xargs tar cf -
+tar cf - ${OUTDIR}

src/python_pachyderm/__init__.py

@@ -91,12 +91,12 @@ def should_import(key):
                     import_item(g, sub_module, key)
 
 
-_import_protos("python_pachyderm.proto.pfs.pfs_pb2")
-_import_protos("python_pachyderm.proto.pps.pps_pb2")
-_import_protos("python_pachyderm.proto.version.versionpb.version_pb2")
-_import_protos("python_pachyderm.proto.transaction.transaction_pb2")
-_import_protos("python_pachyderm.proto.admin.admin_pb2")
-_import_protos("python_pachyderm.proto.auth.auth_pb2")
-_import_protos("python_pachyderm.proto.enterprise.enterprise_pb2")
-_import_protos("python_pachyderm.proto.license.license_pb2")
-_import_protos("python_pachyderm.proto.identity.identity_pb2")
+_import_protos("python_pachyderm.proto.v2.pfs.pfs_pb2")
+_import_protos("python_pachyderm.proto.v2.pps.pps_pb2")
+_import_protos("python_pachyderm.proto.v2.version.versionpb.version_pb2")
+_import_protos("python_pachyderm.proto.v2.transaction.transaction_pb2")
+_import_protos("python_pachyderm.proto.v2.admin.admin_pb2")
+_import_protos("python_pachyderm.proto.v2.auth.auth_pb2")
+_import_protos("python_pachyderm.proto.v2.enterprise.enterprise_pb2")
+_import_protos("python_pachyderm.proto.v2.license.license_pb2")
+_import_protos("python_pachyderm.proto.v2.identity.identity_pb2")

src/python_pachyderm/mixin/admin.py

@@ -1,4 +1,4 @@
-from python_pachyderm.proto.admin import admin_pb2_grpc as admin_grpc
+from python_pachyderm.proto.v2.admin import admin_pb2_grpc as admin_grpc
 from python_pachyderm.service import Service
 
 

src/python_pachyderm/mixin/debug.py

@@ -1,5 +1,5 @@
 from python_pachyderm.service import Service
-from python_pachyderm.proto.debug import debug_pb2 as debug_proto
+from python_pachyderm.proto.v2.debug import debug_pb2 as debug_proto
 
 
 class DebugMixin:

src/python_pachyderm/mixin/health.py

@@ -1,5 +1,5 @@
 from python_pachyderm.service import Service
-from python_pachyderm.proto.health import health_pb2_grpc as health_grpc
+from python_pachyderm.proto.v2.health import health_pb2_grpc as health_grpc
 
 
 class HealthMixin:

src/python_pachyderm/mixin/pfs.py

@@ -3,7 +3,7 @@
 import tarfile
 from contextlib import contextmanager
 
-from python_pachyderm.proto.pfs import pfs_pb2 as pfs_proto
+from python_pachyderm.proto.v2.pfs import pfs_pb2 as pfs_proto
 from python_pachyderm.service import Service
 from .util import commit_from
 

src/python_pachyderm/mixin/pps.py

@@ -4,7 +4,7 @@
 import warnings
 from pathlib import Path
 
-from python_pachyderm.proto.pps import pps_pb2 as pps_proto
+from python_pachyderm.proto.v2.pps import pps_pb2 as pps_proto
 from python_pachyderm.service import Service
 from .util import commit_from
 

src/python_pachyderm/mixin/transaction.py

@@ -1,6 +1,6 @@
 from contextlib import contextmanager
 
-from python_pachyderm.proto.transaction import transaction_pb2 as transaction_proto
+from python_pachyderm.proto.v2.transaction import transaction_pb2 as transaction_proto
 from python_pachyderm.service import Service
 
 

src/python_pachyderm/mixin/util.py

@@ -1,4 +1,4 @@
-from python_pachyderm.proto.pfs import pfs_pb2 as pfs_proto
+from python_pachyderm.proto.v2.pfs import pfs_pb2 as pfs_proto
 
 
 def commit_from(src, allow_just_repo=False):

src/python_pachyderm/mixin/version.py

@@ -1,5 +1,5 @@
 from python_pachyderm.service import Service
-from python_pachyderm.proto.version.versionpb import version_pb2_grpc as version_grpc
+from python_pachyderm.proto.v2.version.versionpb import version_pb2_grpc as version_grpc
 
 
 class VersionMixin: