Add compilation of python schema and service libraries

.bazelrc

@@ -8,3 +8,6 @@ build --host_cxxopt=-std=c++14
 
 # Enable deprecated java_common.compile parameters
 build --incompatible_java_common_parameters=false
+
+# Disabel bazelmod by default
+build --noenable_bzlmod

.bazelversion

@@ -1 +1 @@
-5.4.1
+7.0.0

.github/workflows/build-protobufs-python.yml

@@ -0,0 +1,18 @@
+name: Build OpenSearch Protobufs PYTHON Artifacts
+
+on:
+  pull_request:
+
+jobs:
+  build-protobufs-python:
+    runs-on: ubuntu-latest
+    if: github.repository == 'opensearch-project/opensearch-protobufs'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 21
+      - name: build protobufs python
+        run: bazel build //:python_protos_all

BUILD.bazel

@@ -1,30 +1,25 @@
-# protos/BUILD.bazel
-load("@rules_java//java:defs.bzl", "java_proto_library")
-load("@io_grpc_grpc_java//:java_grpc_library.bzl", "java_grpc_library")
-
 package(default_visibility = ["//visibility:public"])
 
-# Java proto library
-java_proto_library(
-    name = "protos_java",
-    deps = [
-        "//protos/services:document_service_proto",
-        "//protos/services:search_service_proto"
-    ],
-    visibility = ["//visibility:public"],
-)
-
-# Java gRPC library
-java_grpc_library(
-    name = "document_service_grpc_java",
-    srcs = ["//protos/services:document_service_proto"],
-    deps = [":protos_java"],
+java_library(
+    name = "java_protos_all",
     visibility = ["//visibility:public"],
+    exports = [
+        "//protos/schemas:common_java_proto",
+        "//protos/schemas:document_java_proto",
+        "//protos/schemas:search_java_proto",
+        "//protos/services:document_service_java_proto",
+        "//protos/services:search_service_java_proto",
+    ],
 )
 
-java_grpc_library(
-    name = "search_service_grpc_java",
-    srcs = ["//protos/services:search_service_proto"],
-    deps = [":protos_java"],
+py_library(
+    name = "python_protos_all",
     visibility = ["//visibility:public"],
+    deps = [
+        "//protos/schemas:common_python_proto",
+        "//protos/schemas:document_python_proto",
+        "//protos/schemas:search_python_proto",
+        "//protos/services:document_service_python_proto",
+        "//protos/services:search_service_python_proto",
+    ],
 )

CHANGELOG.md

@@ -4,6 +4,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 ### Added
+- Add support for generating python schema and gRPC libraries. ([#114](https://github.com/opensearch-project/opensearch-protobufs/pull/114))
 
 ### Removed
 

DEVELOPER_GUIDE.md

@@ -2,9 +2,22 @@
 Install bazel, using the version in .bazelversion.
 
 # Compile protos and grpc
+
+## All artifacts
 ```
 bazel build //...
 ```
+
+## Java
+```
+bazel build //:java_protos_all
+```
+
+## Python
+```
+bazel build //:python_protos_all
+```
+
 # Proto generated code
 ## Java
 ### Generate Java Code and packaging as a Maven/Gradle dependency
@@ -14,7 +27,7 @@ To package the generated Java files into a Maven-compatible JAR that can be used
 #optional
 rm -rf bazel*
 
-rm -rf generated && bazel build //... && ./tools/java/package_proto_jar.sh
+rm -rf generated && bazel build //:java_protos_all && ./tools/java/package_proto_jar.sh
 ```
 
 This script will:

Dockerfile

@@ -0,0 +1,89 @@
+FROM ubuntu:22.04 AS base-bazel
+
+ENV BAZEL_VERSION=7.0.0
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    git \
+    zip \
+    unzip \
+    openjdk-21-jdk \
+    maven \
+    g++ \
+    gcc \
+    && apt-get clean
+
+RUN apt-get install software-properties-common -y
+RUN add-apt-repository ppa:deadsnakes/ppa -y
+RUN apt-get install python3.10 python3.10-dev -y
+
+RUN curl -fsSL https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh -o bazel-installer.sh \
+    && chmod +x bazel-installer.sh \
+    && ./bazel-installer.sh \
+    && rm bazel-installer.sh
+
+RUN bazel --version
+
+FROM base-bazel AS build-bazel
+
+# Run as non-root - Required for rules_python
+# See: https://github.com/bazelbuild/rules_python/pull/713
+# Create group and user
+RUN groupadd -r bazeluser && useradd -r -m -g bazeluser bazeluser
+RUN mkdir -p /build && \
+    chown -R bazeluser:bazeluser /build && \
+    chmod -R 777 /build
+
+USER bazeluser
+WORKDIR /build
+
+# Copy entire repository for convenience
+# Invalidate cache to ensure updates are captured
+ARG CACHEBUST=1
+COPY --chown=bazeluser:bazeluser . .
+
+#################################################
+##### JAVA STAGES ###############################
+#################################################
+
+FROM build-bazel AS build-bazel-java
+
+RUN bazel build //:java_protos_all
+
+FROM build-bazel-java AS package-bazel-java
+
+RUN /build/tools/java/package_proto_jar.sh
+
+FROM package-bazel-java AS test-bazel-java
+
+ARG OPENSEARCH_BRANCH=main
+ARG PROTO_SNAPSHOT_VERSION=0.4.0
+
+ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64
+ENV PATH=$PATH:$JAVA_HOME/bin
+ENV PUB_ARTIFACTS=/build/generated/maven/publish/
+
+RUN mvn install:install-file \
+  -Dfile=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT.jar \
+  -DpomFile=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT.pom \
+  -Dsources=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT-sources.jar \
+  -Djavadoc=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT-javadoc.jar
+
+RUN git clone --branch ${OPENSEARCH_BRANCH} https://github.com/opensearch-project/OpenSearch.git /build/opensearch
+
+WORKDIR /build/opensearch
+
+# Update transport-grpc/build.gradle to use PROTO_SNAPSHOT_VERSION
+RUN sed -i 's/org\.opensearch:protobufs:[0-9]\+\.[0-9]\+\.[0-9]\+/org.opensearch:protobufs:'"${PROTO_SNAPSHOT_VERSION}"'/' /build/opensearch/plugins/transport-grpc/build.gradle
+
+RUN ./gradlew :plugins:transport-grpc:test -Drepos.mavenLocal
+RUN ./gradlew :plugins:transport-grpc:internalClusterTest -Drepos.mavenLocal
+
+#################################################
+##### PYTHON STAGES #############################
+#################################################
+
+FROM build-bazel AS build-bazel-python
+
+RUN bazel build //:python_protos_all

WORKSPACE

@@ -2,7 +2,33 @@ workspace(name = "proto_workspace")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-# Protocol Buffers dependencies
+"""
+The version of rules_apple pulled in by com_github_grpc_grpc is missing a bugfix required
+for bazel 7.x support. "'apple_common' value has no field or method 'multi_arch_split'".
+See release notes: https://github.com/bazelbuild/rules_apple/releases/tag/3.0.0-rc1.
+"""
+
+http_archive(
+    name = "build_bazel_rules_apple",
+    sha256 = "20da675977cb8249919df14d0ce6165d7b00325fb067f0b06696b893b90a55e8",
+    url = "https://github.com/bazelbuild/rules_apple/releases/download/3.0.0/rules_apple.3.0.0.tar.gz",
+)
+
+"""
+Protoc compiler - 3.25.5 and associated C dependencies.
+Includes some native support for language specific rules.
+"""
+
+http_archive(
+    name = "com_google_absl",
+    sha256 = "f50e5ac311a81382da7fa75b97310e4b9006474f9560ac46f54a9967f07d4ae3",
+    strip_prefix = "abseil-cpp-20240722.0",
+    urls = [
+        "https://storage.googleapis.com/grpc-bazel-mirror/github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz",
+        "https://github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz",
+    ],
+)
+
 http_archive(
     name = "com_google_protobuf",
     sha256 = "4356e78744dfb2df3890282386c8568c85868116317d9b3ad80eb11c2aecf2ff",
@@ -11,65 +37,80 @@ http_archive(
 )
 
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
-
 protobuf_deps()
 
-# Java rules
+"""
+Pin compatible version of go language rules.
+Overrides the version set in com_github_grpc_grpc.
+"""
+
 http_archive(
-    name = "rules_java",
-    sha256 = "f8ae9ed3887df02f40de9f4f7ac3873e6dd7a471f9cddf63952538b94b59aeb3",
+    name = "io_bazel_rules_go",
+    sha256 = "6734a719993b1ba4ebe9806e853864395a8d3968ad27f9dd759c196b3eb3abe8",
     urls = [
-        "https://github.com/bazelbuild/rules_java/releases/download/7.6.1/rules_java-7.6.1.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.45.1/rules_go-v0.45.1.zip",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.45.1/rules_go-v0.45.1.zip",
     ],
 )
 
-# load("@rules_java//java:repositories.bzl", "rules_java_dependencies")
-# rules_java_dependencies()
+"""
+Language and gRPC rules from the actively maintained rules-proto-grpc project.
+Java language/gRPC rules.
+Python language/gRPC rules.
+"""
 
-# Proto rules
 http_archive(
-    name = "rules_proto",
-    sha256 = "37005d35aba73a843b34f194c0c41633a0b2fd90f8b84c91b391b995ffc593f2",
-    strip_prefix = "rules_proto-5.3.0-21.7",
-    urls = ["https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.zip"],
+    name = "rules_proto_grpc",
+    sha256 = "9ba7299c5eb6ec45b6b9a0ceb9916d0ab96789ac8218269322f0124c0c0d24e2",
+    strip_prefix = "rules_proto_grpc-4.5.0",
+    urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/releases/download/4.5.0/rules_proto_grpc-4.5.0.tar.gz"],
 )
 
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
-
-rules_proto_dependencies()
+load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains")
+rules_proto_grpc_repos()
+rules_proto_grpc_toolchains()
 
-rules_proto_toolchains()
+"""
+Official gRPC repo bazel dependencies.
+We must match the version used in OS core exactly - 1.68.2.
+"""
 
-# gRPC dependencies
 http_archive(
-    name = "io_grpc_grpc_java",
-    sha256 = "dc1ad2272c1442075c59116ec468a7227d0612350c44401237facd35aab15732",
-    strip_prefix = "grpc-java-1.68.2",
-    urls = ["https://github.com/grpc/grpc-java/archive/v1.68.2.tar.gz"],
+    name = "com_github_grpc_grpc",
+    strip_prefix = "grpc-1.68.2",
+    urls = ["https://github.com/grpc/grpc/archive/v1.68.2.tar.gz"],
+    sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c",
 )
 
-load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_ARTIFACTS", "IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS", "grpc_java_repositories")
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
+grpc_deps()
 
-grpc_java_repositories()
+load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
+grpc_extra_deps()
 
-# Use rules_jvm_external to manage Maven dependencies
-http_archive(
-    name = "rules_jvm_external",
-    sha256 = "62133c125bf4109dfd9d2af64830208356ce4ef8b165a6ef15bbff7460b35c3a",
-    strip_prefix = "rules_jvm_external-3.0",
-    url = "https://github.com/bazelbuild/rules_jvm_external/archive/3.0.zip",
-)
+"""
+Load language/gRPC rules last to ensure we pick up the correct protobuf and gRPC versions.
+"""
+
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+rules_proto_dependencies()
+rules_proto_toolchains()
+
+load("@rules_proto_grpc//python:repositories.bzl", rules_proto_grpc_python_repos = "python_repos")
+rules_proto_grpc_python_repos()
+
+load("@rules_proto_grpc//java:repositories.bzl", rules_proto_grpc_java_repos = "java_repos")
+rules_proto_grpc_java_repos()
+
+"""
+Dependencies for java gRPC rules are sourced from maven. 
+"""
 
 load("@rules_jvm_external//:defs.bzl", "maven_install")
+load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_ARTIFACTS", "IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS", "grpc_java_repositories")
 
 maven_install(
-    artifacts = IO_GRPC_GRPC_JAVA_ARTIFACTS + [
-        "io.netty:netty-handler:4.1.118.Final",
-        "commons-codec:commons-codec:1.13",
-        "org.jetbrains.kotlin:kotlin-stdlib:1.6.0",
-        "com.google.protobuf:protobuf-java:3.25.5",
-        "com.squareup.okio:okio:3.4.0",
-    ],
+    artifacts = IO_GRPC_GRPC_JAVA_ARTIFACTS,
     generate_compat_repositories = True,
     override_targets = IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS,
     repositories = [
@@ -78,5 +119,5 @@ maven_install(
 )
 
 load("@maven//:compat.bzl", "compat_repositories")
-
 compat_repositories()
+grpc_java_repositories()