diff --git a/.bazelrc b/.bazelrc index 05dc574c..b99a8da1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -8,3 +8,6 @@ build --host_cxxopt=-std=c++14 # Enable deprecated java_common.compile parameters build --incompatible_java_common_parameters=false + +# Disabel bazelmod by default +build --noenable_bzlmod diff --git a/.bazelversion b/.bazelversion index ade65226..66ce77b7 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -5.4.1 +7.0.0 diff --git a/.github/workflows/build-protobufs-python.yml b/.github/workflows/build-protobufs-python.yml new file mode 100644 index 00000000..aac2131d --- /dev/null +++ b/.github/workflows/build-protobufs-python.yml @@ -0,0 +1,18 @@ +name: Build OpenSearch Protobufs PYTHON Artifacts + +on: + pull_request: + +jobs: + build-protobufs-python: + runs-on: ubuntu-latest + if: github.repository == 'opensearch-project/opensearch-protobufs' + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: 21 + - name: build protobufs python + run: bazel build //:python_protos_all diff --git a/BUILD.bazel b/BUILD.bazel index 664fd1e1..bf56d6fa 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,30 +1,25 @@ -# protos/BUILD.bazel -load("@rules_java//java:defs.bzl", "java_proto_library") -load("@io_grpc_grpc_java//:java_grpc_library.bzl", "java_grpc_library") - package(default_visibility = ["//visibility:public"]) -# Java proto library -java_proto_library( - name = "protos_java", - deps = [ - "//protos/services:document_service_proto", - "//protos/services:search_service_proto" - ], - visibility = ["//visibility:public"], -) - -# Java gRPC library -java_grpc_library( - name = "document_service_grpc_java", - srcs = ["//protos/services:document_service_proto"], - deps = [":protos_java"], +java_library( + name = "java_protos_all", visibility = ["//visibility:public"], + exports = [ + "//protos/schemas:common_java_proto", + "//protos/schemas:document_java_proto", + "//protos/schemas:search_java_proto", + "//protos/services:document_service_java_proto", + "//protos/services:search_service_java_proto", + ], ) -java_grpc_library( - name = "search_service_grpc_java", - srcs = ["//protos/services:search_service_proto"], - deps = [":protos_java"], +py_library( + name = "python_protos_all", visibility = ["//visibility:public"], + deps = [ + "//protos/schemas:common_python_proto", + "//protos/schemas:document_python_proto", + "//protos/schemas:search_python_proto", + "//protos/services:document_service_python_proto", + "//protos/services:search_service_python_proto", + ], ) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06843445..27678176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] ### Added +- Add support for generating python schema and gRPC libraries. ([#114](https://github.com/opensearch-project/opensearch-protobufs/pull/114)) ### Removed diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e5ec6c3f..d4e5612d 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2,9 +2,22 @@ Install bazel, using the version in .bazelversion. # Compile protos and grpc + +## All artifacts ``` bazel build //... ``` + +## Java +``` +bazel build //:java_protos_all +``` + +## Python +``` +bazel build //:python_protos_all +``` + # Proto generated code ## Java ### Generate Java Code and packaging as a Maven/Gradle dependency @@ -14,7 +27,7 @@ To package the generated Java files into a Maven-compatible JAR that can be used #optional rm -rf bazel* -rm -rf generated && bazel build //... && ./tools/java/package_proto_jar.sh +rm -rf generated && bazel build //:java_protos_all && ./tools/java/package_proto_jar.sh ``` This script will: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..5d8b1d9a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,89 @@ +FROM ubuntu:22.04 AS base-bazel + +ENV BAZEL_VERSION=7.0.0 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + curl \ + git \ + zip \ + unzip \ + openjdk-21-jdk \ + maven \ + g++ \ + gcc \ + && apt-get clean + +RUN apt-get install software-properties-common -y +RUN add-apt-repository ppa:deadsnakes/ppa -y +RUN apt-get install python3.10 python3.10-dev -y + +RUN curl -fsSL https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh -o bazel-installer.sh \ + && chmod +x bazel-installer.sh \ + && ./bazel-installer.sh \ + && rm bazel-installer.sh + +RUN bazel --version + +FROM base-bazel AS build-bazel + +# Run as non-root - Required for rules_python +# See: https://github.com/bazelbuild/rules_python/pull/713 +# Create group and user +RUN groupadd -r bazeluser && useradd -r -m -g bazeluser bazeluser +RUN mkdir -p /build && \ + chown -R bazeluser:bazeluser /build && \ + chmod -R 777 /build + +USER bazeluser +WORKDIR /build + +# Copy entire repository for convenience +# Invalidate cache to ensure updates are captured +ARG CACHEBUST=1 +COPY --chown=bazeluser:bazeluser . . + +################################################# +##### JAVA STAGES ############################### +################################################# + +FROM build-bazel AS build-bazel-java + +RUN bazel build //:java_protos_all + +FROM build-bazel-java AS package-bazel-java + +RUN /build/tools/java/package_proto_jar.sh + +FROM package-bazel-java AS test-bazel-java + +ARG OPENSEARCH_BRANCH=main +ARG PROTO_SNAPSHOT_VERSION=0.4.0 + +ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64 +ENV PATH=$PATH:$JAVA_HOME/bin +ENV PUB_ARTIFACTS=/build/generated/maven/publish/ + +RUN mvn install:install-file \ + -Dfile=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT.jar \ + -DpomFile=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT.pom \ + -Dsources=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT-sources.jar \ + -Djavadoc=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT-javadoc.jar + +RUN git clone --branch ${OPENSEARCH_BRANCH} https://github.com/opensearch-project/OpenSearch.git /build/opensearch + +WORKDIR /build/opensearch + +# Update transport-grpc/build.gradle to use PROTO_SNAPSHOT_VERSION +RUN sed -i 's/org\.opensearch:protobufs:[0-9]\+\.[0-9]\+\.[0-9]\+/org.opensearch:protobufs:'"${PROTO_SNAPSHOT_VERSION}"'/' /build/opensearch/plugins/transport-grpc/build.gradle + +RUN ./gradlew :plugins:transport-grpc:test -Drepos.mavenLocal +RUN ./gradlew :plugins:transport-grpc:internalClusterTest -Drepos.mavenLocal + +################################################# +##### PYTHON STAGES ############################# +################################################# + +FROM build-bazel AS build-bazel-python + +RUN bazel build //:python_protos_all diff --git a/WORKSPACE b/WORKSPACE index aad88377..05fada97 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,7 +2,33 @@ workspace(name = "proto_workspace") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -# Protocol Buffers dependencies +""" +The version of rules_apple pulled in by com_github_grpc_grpc is missing a bugfix required +for bazel 7.x support. "'apple_common' value has no field or method 'multi_arch_split'". +See release notes: https://github.com/bazelbuild/rules_apple/releases/tag/3.0.0-rc1. +""" + +http_archive( + name = "build_bazel_rules_apple", + sha256 = "20da675977cb8249919df14d0ce6165d7b00325fb067f0b06696b893b90a55e8", + url = "https://github.com/bazelbuild/rules_apple/releases/download/3.0.0/rules_apple.3.0.0.tar.gz", +) + +""" +Protoc compiler - 3.25.5 and associated C dependencies. +Includes some native support for language specific rules. +""" + +http_archive( + name = "com_google_absl", + sha256 = "f50e5ac311a81382da7fa75b97310e4b9006474f9560ac46f54a9967f07d4ae3", + strip_prefix = "abseil-cpp-20240722.0", + urls = [ + "https://storage.googleapis.com/grpc-bazel-mirror/github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz", + "https://github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz", + ], +) + http_archive( name = "com_google_protobuf", sha256 = "4356e78744dfb2df3890282386c8568c85868116317d9b3ad80eb11c2aecf2ff", @@ -11,65 +37,80 @@ http_archive( ) load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") - protobuf_deps() -# Java rules +""" +Pin compatible version of go language rules. +Overrides the version set in com_github_grpc_grpc. +""" + http_archive( - name = "rules_java", - sha256 = "f8ae9ed3887df02f40de9f4f7ac3873e6dd7a471f9cddf63952538b94b59aeb3", + name = "io_bazel_rules_go", + sha256 = "6734a719993b1ba4ebe9806e853864395a8d3968ad27f9dd759c196b3eb3abe8", urls = [ - "https://github.com/bazelbuild/rules_java/releases/download/7.6.1/rules_java-7.6.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.45.1/rules_go-v0.45.1.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.45.1/rules_go-v0.45.1.zip", ], ) -# load("@rules_java//java:repositories.bzl", "rules_java_dependencies") -# rules_java_dependencies() +""" +Language and gRPC rules from the actively maintained rules-proto-grpc project. +Java language/gRPC rules. +Python language/gRPC rules. +""" -# Proto rules http_archive( - name = "rules_proto", - sha256 = "37005d35aba73a843b34f194c0c41633a0b2fd90f8b84c91b391b995ffc593f2", - strip_prefix = "rules_proto-5.3.0-21.7", - urls = ["https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.zip"], + name = "rules_proto_grpc", + sha256 = "9ba7299c5eb6ec45b6b9a0ceb9916d0ab96789ac8218269322f0124c0c0d24e2", + strip_prefix = "rules_proto_grpc-4.5.0", + urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/releases/download/4.5.0/rules_proto_grpc-4.5.0.tar.gz"], ) -load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") - -rules_proto_dependencies() +load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains") +rules_proto_grpc_repos() +rules_proto_grpc_toolchains() -rules_proto_toolchains() +""" +Official gRPC repo bazel dependencies. +We must match the version used in OS core exactly - 1.68.2. +""" -# gRPC dependencies http_archive( - name = "io_grpc_grpc_java", - sha256 = "dc1ad2272c1442075c59116ec468a7227d0612350c44401237facd35aab15732", - strip_prefix = "grpc-java-1.68.2", - urls = ["https://github.com/grpc/grpc-java/archive/v1.68.2.tar.gz"], + name = "com_github_grpc_grpc", + strip_prefix = "grpc-1.68.2", + urls = ["https://github.com/grpc/grpc/archive/v1.68.2.tar.gz"], + sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", ) -load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_ARTIFACTS", "IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS", "grpc_java_repositories") +load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") +grpc_deps() -grpc_java_repositories() +load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") +grpc_extra_deps() -# Use rules_jvm_external to manage Maven dependencies -http_archive( - name = "rules_jvm_external", - sha256 = "62133c125bf4109dfd9d2af64830208356ce4ef8b165a6ef15bbff7460b35c3a", - strip_prefix = "rules_jvm_external-3.0", - url = "https://github.com/bazelbuild/rules_jvm_external/archive/3.0.zip", -) +""" +Load language/gRPC rules last to ensure we pick up the correct protobuf and gRPC versions. +""" + +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") +rules_proto_dependencies() +rules_proto_toolchains() + +load("@rules_proto_grpc//python:repositories.bzl", rules_proto_grpc_python_repos = "python_repos") +rules_proto_grpc_python_repos() + +load("@rules_proto_grpc//java:repositories.bzl", rules_proto_grpc_java_repos = "java_repos") +rules_proto_grpc_java_repos() + +""" +Dependencies for java gRPC rules are sourced from maven. +""" load("@rules_jvm_external//:defs.bzl", "maven_install") +load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_ARTIFACTS", "IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS", "grpc_java_repositories") maven_install( - artifacts = IO_GRPC_GRPC_JAVA_ARTIFACTS + [ - "io.netty:netty-handler:4.1.118.Final", - "commons-codec:commons-codec:1.13", - "org.jetbrains.kotlin:kotlin-stdlib:1.6.0", - "com.google.protobuf:protobuf-java:3.25.5", - "com.squareup.okio:okio:3.4.0", - ], + artifacts = IO_GRPC_GRPC_JAVA_ARTIFACTS, generate_compat_repositories = True, override_targets = IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS, repositories = [ @@ -78,5 +119,5 @@ maven_install( ) load("@maven//:compat.bzl", "compat_repositories") - compat_repositories() +grpc_java_repositories() diff --git a/guava.BUILD b/guava.BUILD deleted file mode 100644 index 621932b9..00000000 --- a/guava.BUILD +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -java_import( - name = "jar", - jars = ["guava-31.0.1-jre.jar"], -) diff --git a/jsr305.BUILD b/jsr305.BUILD deleted file mode 100644 index 85f1aad7..00000000 --- a/jsr305.BUILD +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -java_import( - name = "jar", - jars = ["jsr305-3.0.2.jar"], -) diff --git a/protos/schemas/BUILD.bazel b/protos/schemas/BUILD.bazel index 5bd00023..39010cac 100644 --- a/protos/schemas/BUILD.bazel +++ b/protos/schemas/BUILD.bazel @@ -1,14 +1,14 @@ # protos/schemas/BUILD.bazel load("@rules_proto//proto:defs.bzl", "proto_library") -load("@rules_java//java:defs.bzl", "java_proto_library") +load("@rules_proto_grpc//java:defs.bzl", "java_proto_library") +load("@rules_proto_grpc//python:defs.bzl", "python_proto_library") + package(default_visibility = ["//visibility:public"]) proto_library( name = "common_proto", srcs = ["common.proto"], - deps = [ - "@com_google_protobuf//:struct_proto", - ], + deps = ["@com_google_protobuf//:struct_proto"], ) proto_library( @@ -26,21 +26,37 @@ proto_library( deps = [ ":common_proto", "@com_google_protobuf//:struct_proto", - ], + ], ) -# Java proto libraries java_proto_library( name = "common_java_proto", - deps = [":common_proto"], + protos = [":common_proto"], ) java_proto_library( name = "document_java_proto", - deps = [":document_proto"], + protos = [":document_proto"], + deps = [":common_java_proto"], ) java_proto_library( name = "search_java_proto", - deps = [":search_proto"], + protos = [":search_proto"], + deps = [":common_java_proto"], +) + +python_proto_library( + name = "common_python_proto", + protos = [":common_proto"], +) + +python_proto_library( + name = "document_python_proto", + protos = [":document_proto"], +) + +python_proto_library( + name = "search_python_proto", + protos = [":search_proto"], ) diff --git a/protos/services/BUILD.bazel b/protos/services/BUILD.bazel index 3aa767c9..aab06b76 100644 --- a/protos/services/BUILD.bazel +++ b/protos/services/BUILD.bazel @@ -1,44 +1,40 @@ # protos/services/BUILD.bazel load("@rules_proto//proto:defs.bzl", "proto_library") -load("@rules_java//java:defs.bzl", "java_proto_library") -load("@io_grpc_grpc_java//:java_grpc_library.bzl", "java_grpc_library") +load("@rules_proto_grpc//java:defs.bzl", "java_grpc_library") +load("@rules_proto_grpc//python:defs.bzl", "python_grpc_library") package(default_visibility = ["//visibility:public"]) proto_library( name = "document_service_proto", srcs = ["document_service.proto"], - deps = [ - "//protos/schemas:document_proto", - ], + deps = ["//protos/schemas:document_proto"], ) proto_library( name = "search_service_proto", srcs = ["search_service.proto"], - deps = [ - "//protos/schemas:search_proto", - ], + deps = ["//protos/schemas:search_proto"], ) -# Java proto library -java_proto_library( - name = "services_java_proto", - deps = [ - ":document_service_proto", - ":search_service_proto" - ], +java_grpc_library( + name = "document_service_java_proto", + protos = [":document_service_proto"], + deps = ["//protos/schemas:document_java_proto"], ) -# Java gRPC library java_grpc_library( - name = "document_service_grpc_java", - srcs = [":document_service_proto"], - deps = [":services_java_proto"], + name = "search_service_java_proto", + protos = [":search_service_proto"], + deps = ["//protos/schemas:search_java_proto"], ) -java_grpc_library( - name = "search_service_grpc_java", - srcs = [":search_service_proto"], - deps = [":services_java_proto"], +python_grpc_library( + name = "document_service_python_proto", + protos = [":document_service_proto"], +) + +python_grpc_library( + name = "search_service_python_proto", + protos = [":search_service_proto"], ) diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel deleted file mode 100644 index 766af4ee..00000000 --- a/third_party/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -# Empty BUILD file to create the third_party package diff --git a/third_party/guava.BUILD b/third_party/guava.BUILD deleted file mode 100644 index 9d5ccd92..00000000 --- a/third_party/guava.BUILD +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -java_import( - name = "jar", - jars = ["guava-33.2.1-jre.jar"], -) diff --git a/third_party/jsr305.BUILD b/third_party/jsr305.BUILD deleted file mode 100644 index 85f1aad7..00000000 --- a/third_party/jsr305.BUILD +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -java_import( - name = "jar", - jars = ["jsr305-3.0.2.jar"], -) diff --git a/tools/java/generate_java.sh b/tools/java/generate_java.sh index e7db41f8..7bc34a2c 100755 --- a/tools/java/generate_java.sh +++ b/tools/java/generate_java.sh @@ -10,25 +10,22 @@ function exit_script() { # Trap abnormal exit due to bazel not return non-zero status trap exit_script TERM INT -# Set up variables +# Build the Java proto library ROOT_DIR="`dirname "$(realpath $0)"`/../.." -OUTPUT_DIR="$ROOT_DIR/generated/java" -BAZEL_BIN_DIR=$(readlink -f bazel-bin) +echo "Building Java proto library..." +cd $ROOT_DIR && bazel build //:java_protos_all && cd - > /dev/null # Clean up existing output directory +OUTPUT_DIR="$ROOT_DIR/generated/java" rm -rf "$OUTPUT_DIR" mkdir -p "$OUTPUT_DIR" -# Build the Java proto library -echo "Building Java proto library..." -cd $ROOT_DIR && bazel build //... && cd - > /dev/null - # Find all source JAR files echo "Finding source JAR files..." -SRC_JARS=$(find "$BAZEL_BIN_DIR"/ -name "*-speed-src.jar") -SRC_JARS+=" " -SRC_JARS+=$(find "$BAZEL_BIN_DIR"/ -name "*grpc_java-src.jar") +cd $ROOT_DIR && BAZEL_BIN_DIR=$(readlink -f bazel-bin) +SRC_JARS=$(find "$BAZEL_BIN_DIR"/ -name "*_java_proto*.srcjar") echo $SRC_JARS + # Extract Java files from source JAR files echo "Extracting Java files from source JAR files..." for jar in $SRC_JARS; do