xds: bazel proto gen code generation scripts (#2775)

balancer/xds/internal/regenerate_scripts/envoy-proto-gen.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -ex
+DATA_PLANE_API_VERSION=1935b52f94f7889ad9f538a17250e78cffd0af27
+
+git clone [email protected]:envoyproxy/data-plane-api.git
+git clone [email protected]:envoyproxy/protoc-gen-validate.git
+
+cd data-plane-api
+git checkout $DATA_PLANE_API_VERSION
+cp ../utils/WORKSPACE .
+bazel clean --expunge
+
+# We download a local copy of the protoc-gen-validate repo to be used by bazel
+# for customizing proto generated code import path.
+# And we do a simple grep here to get the release version of the
+# proto-gen-validate that gets used by data-plane-api.
+PROTOC_GEN_VALIDATE=v$(grep "PGV_RELEASE =" ./bazel/repository_locations.bzl  | sed -r 's/.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/')
+
+cd ../protoc-gen-validate
+git checkout $PROTOC_GEN_VALIDATE
+git apply ../utils/protoc-gen-validate.patch
+
+cd ../data-plane-api
+
+# cleanup.sh remove all gogo proto related imports and labels.
+../utils/cleanup.sh
+
+git apply ../utils/data-plane-api.patch
+# proto-gen.sh build all packages required for grpc xds implementation and move
+# proto generated code to grpc/balancer/xds/internal/proto subdirectory.
+../utils/proto-gen.sh
+

balancer/xds/internal/regenerate_scripts/utils/README.md

@@ -0,0 +1,2 @@
+Run ./envoy-proto-gen.sh to generate xds proto generated code for grpc usage.
+Make sure your $GOPATH is valid and you have grpc-go repo locally.

balancer/xds/internal/regenerate_scripts/utils/WORKSPACE

@@ -0,0 +1,57 @@
+workspace(name = "envoy_api")
+
+local_repository(
+    name = "com_lyft_protoc_gen_validate",
+    path = "../protoc-gen-validate",
+)
+
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "io_bazel_rules_go",
+    urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.17.1/rules_go-0.17.1.tar.gz"],
+    sha256 = "6776d68ebb897625dead17ae510eac3d5f6342367327875210df44dbe2aeeb19",
+)
+
+git_repository(
+    name = "com_github_golang_protobuf",
+    remote = "https://github.com/golang/protobuf",
+    commit = "aa810b61a9c79d51363740d207bb46cf8e620ed5",
+    shallow_since = "1534281267 -0700",
+    patches = [
+        "@io_bazel_rules_go//third_party:com_github_golang_protobuf-gazelle.patch",
+        "@io_bazel_rules_go//third_party:com_github_golang_protobuf-extras.patch",
+    ],
+    patch_args = ["-p1"],
+)
+
+load("//bazel:repositories.bzl", "api_dependencies")
+
+api_dependencies()
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains")
+go_rules_dependencies()
+go_register_toolchains()
+
+http_archive(
+    name = "bazel_gazelle",
+    urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.17.0/bazel-gazelle-0.17.0.tar.gz"],
+    sha256 = "3c681998538231a2d24d0c07ed5a7658cb72bfb5fd4bf9911157c0e9ac6a2687",
+)
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+gazelle_dependencies()
+
+bind(
+    name = "six",
+    actual = "@six_archive//:six",
+)
+
+http_archive(
+    name = "six_archive",
+    sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
+    build_file = "@com_google_protobuf//:six.BUILD",
+    url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55",
+)
+

balancer/xds/internal/regenerate_scripts/utils/cleanup.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+find . -name '*.proto' -print0 |
+while IFS= read -r -d '' f
+do
+  commands=(
+    # Import mangling.
+    -e 's#import "gogoproto/gogo.proto";##'
+    # Remove references to gogo.proto extensions.
+    -e 's#option (gogoproto\.[a-z_]\+) = \(true\|false\);##'
+    -e 's#\(, \)\?(gogoproto\.[a-z_]\+) = \(true\|false\),\?##'
+    # gogoproto removal can result in empty brackets.
+    -e 's# \[\]##'
+    # gogoproto removal can result in four spaces on a line by itself.
+    -e '/^    $/d'
+  )
+  sed -i "${commands[@]}" "$f"
+
+  # gogoproto removal can leave a comma on the last element in a list.
+  # This needs to run separately after all the commands above have finished
+  # since it is multi-line and rewrites the output of the above patterns.
+  sed -i -e '$!N; s#\(.*\),\([[:space:]]*\];\)#\1\2#; t; P; D;' "$f"
+done

balancer/xds/internal/regenerate_scripts/utils/data-plane-api.patch

@@ -0,0 +1,31 @@
+diff --git a/bazel/api_build_system.bzl b/bazel/api_build_system.bzl
+index c68ccbd..e6cc8cb 100644
+--- a/bazel/api_build_system.bzl
++++ b/bazel/api_build_system.bzl
+@@ -7,7 +7,7 @@ _PY_SUFFIX = "_py"
+ _CC_SUFFIX = "_cc"
+ _GO_PROTO_SUFFIX = "_go_proto"
+ _GO_GRPC_SUFFIX = "_go_grpc"
+-_GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/data-plane-api/api/"
++_GO_IMPORTPATH_PREFIX = "google.golang.org/grpc/balancer/xds/internal/proto/"
+
+ def _Suffix(d, suffix):
+     return d + suffix
+@@ -42,7 +42,7 @@ def api_py_proto_library(name, srcs = [], deps = [], has_services = 0):
+ def api_go_proto_library(name, proto, deps = []):
+     go_proto_library(
+         name = _Suffix(name, _GO_PROTO_SUFFIX),
+-        importpath = _Suffix(_GO_IMPORTPATH_PREFIX, name),
++        importpath = _Suffix(_GO_IMPORTPATH_PREFIX + native.package_name() + "/", name),
+         proto = proto,
+         visibility = ["//visibility:public"],
+         deps = deps + [
+@@ -60,7 +60,7 @@ def api_go_proto_library(name, proto, deps = []):
+ def api_go_grpc_library(name, proto, deps = []):
+     go_grpc_library(
+         name = _Suffix(name, _GO_GRPC_SUFFIX),
+-        importpath = _Suffix(_GO_IMPORTPATH_PREFIX, name),
++        importpath = _Suffix(_GO_IMPORTPATH_PREFIX + native.package_name() + "/", name),
+         proto = proto,
+         visibility = ["//visibility:public"],
+         deps = deps + [

balancer/xds/internal/regenerate_scripts/utils/proto-gen.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# packages is the collection of the packages that are required by xds for grpc.
+packages=(
+  envoy/service/discovery/v2:ads_go_grpc
+  envoy/api/v2:eds_go_grpc
+  envoy/api/v2:cds_go_grpc
+  envoy/api/v2/core:address_go_proto
+  envoy/api/v2/core:base_go_proto
+  envoy/api/v2/endpoint:endpoint_go_proto
+  envoy/type:percent_go_proto
+  envoy/service/load_stats/v2:lrs_go_grpc
+  udpa/data/orca/v1:orca_load_report_go_proto
+  udpa/service/orca/v1:orca_go_grpc
+)
+
+if [ -z $GOPATH ]; then echo 'empty $GOPATH, exiting.'; exit 1
+fi
+
+for i in ${packages[@]}
+do
+  bazel build "$i"
+done
+
+dest="$PWD/../../proto/"
+rm -rf "$dest"
+srcs=(
+    "find -L ./bazel-bin/envoy/ -name *.pb.go -print0"
+    "find -L ./bazel-bin/udpa/ -name *.pb.go -print0"
+    "find -L ./bazel-bin/ -name validate.pb.go -print0"
+)
+
+for src in "${srcs[@]}"
+do
+    eval "$src" |
+    while IFS= read -r -d '' origin
+    do
+        target="${origin##*proto/}"
+        final="$dest$target"
+        mkdir -p "${final%*/*}"
+        cp "$origin" "$dest$target"
+    done
+done

balancer/xds/internal/regenerate_scripts/utils/protoc-gen-validate.patch

@@ -0,0 +1,26 @@
+diff --git a/validate/BUILD b/validate/BUILD
+index af8c6c1..939d997 100644
+--- a/validate/BUILD
++++ b/validate/BUILD
+@@ -31,7 +31,7 @@ py_proto_library(
+
+ go_proto_library(
+     name = "go_default_library",
+-    importpath = "github.com/lyft/protoc-gen-validate/validate",
++    importpath = "google.golang.org/grpc/balancer/xds/internal/proto/validate",
+     proto = ":validate_proto",
+     visibility = ["//visibility:public"],
+ )
+diff --git a/validate/validate.proto b/validate/validate.proto
+index 1c5e04a..7f5d4b0 100644
+--- a/validate/validate.proto
++++ b/validate/validate.proto
+@@ -1,7 +1,7 @@
+ syntax = "proto2";
+ package validate;
+
+-option go_package = "github.com/lyft/protoc-gen-validate/validate";
++option go_package = "google.golang.org/grpc/balancer/xds/internal/proto/validate";
+ option java_package = "com.lyft.pgv.validate";
+
+ import "google/protobuf/descriptor.proto";