Skip to content

Commit

Permalink
Upgrade everything!
Browse files Browse the repository at this point in the history 
What started out as an attempt to upgrade gRPC has snowballed into this monster...

* Upgrades gRPC to the latest version, including the Java bindings from `grpc-java`
  * `grpc-java` actually specifies [Maven overrides](https://github.com/bazelbuild/bazel-central-registry/blob/353b485c6cd965062cced0d40496b70e606e2dbe/modules/grpc-java/1.66.0/MODULE.bazel#L88-L201), causing rules_jvm_external's references to `io.grpc.*` and `com.google.protobuf.*` to alias to actual Bazel build targets in `@grpc-java` and `@protobuf`. So this effectively means we're no longer using the gRPC jars from Maven
  * To support these overrides, our custom patch for `rules_jvm_external` had to be updated to include the `alias` targets in the `BUILD.vendor` file.
* The upgrade to gRPC means we now use the latest googleapis from BCR
  * This means we can remove the vendored copy of those protos. Updated all references to googleapis protos across the codebase.
* The upgrade to gRPC means we now use the latest version of protobuf
  * This ran into some trouble with the concurrent work to move Bazel's proto rules into protobuf, which means that we currently have to use the protobuf version from HEAD.
  * We also have to patch out protobuf's dependency on rules_rust, as it introduces a lot of dependency bloat and makes it impossible to build a bootstrap distribution archive
  * The dependency on upb is completely removed.

Fixes #22719

Closes #23767.

PiperOrigin-RevId: 681163732
Change-Id: Ifd7476833f61c40483850a6a825f79537ffad74a
  • Loading branch information
@Wyverald @copybara-github
Wyverald authored and copybara-github committed Oct 1, 2024
1 parent 4720b4c commit c46b9ce
Show file tree
Hide file tree
Showing 40 changed files with 3,673 additions and 2,075 deletions.
1 change: 1 addition & 0 deletions .bazelrc
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
common --enable_platform_specific_config
common --incompatible_use_plus_in_repo_names
common --experimental_google_legacy_api # needed by protobuf

# Shared configuration flags to build and test Bazel on RBE.
build:remote_shared --remote_instance_name=projects/bazel-untrusted/instances/default_instance
Expand Down
12 changes: 4 additions & 8 deletions BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,16 +93,11 @@ genrule(
name = "generate_dist_lockfile",
srcs = [
"MODULE.bazel",
"//third_party/googleapis:MODULE.bazel",
"//third_party/remoteapis:MODULE.bazel",
"//third_party:BUILD",
"//third_party:rules_jvm_external_6.0.patch",
"//third_party:protobuf_21.7.patch",
"//third_party/upb:BUILD",
"//third_party/upb:00_remove_toolchain_transition.patch",
"//third_party/upb:01_remove_werror.patch",
"//third_party/grpc:BUILD",
"//third_party/grpc:00_disable_layering_check.patch",
"//third_party/protobuf:BUILD",
"//third_party/protobuf:remove_rules_rust.patch",
],
outs = ["MODULE.bazel.lock.dist"],
cmd = " && ".join([
Expand All @@ -126,7 +121,8 @@ genrule(
pkg_tar(
name = "bootstrap-jars",
srcs = [
"@blake3",
"//third_party/googleapis:dist_jars",
"//third_party/grpc-java:grpc_jars",
"@com_google_protobuf//:protobuf_java",
"@com_google_protobuf//:protobuf_java_util",
"@com_google_protobuf//:protobuf_javalite",
Expand Down
143 changes: 68 additions & 75 deletions MODULE.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@ module(
# Bazel module dependencies
# =========================================

bazel_dep(name = "rules_license", version = "0.0.7")
bazel_dep(name = "rules_license", version = "1.0.0")
bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
bazel_dep(name = "grpc", version = "1.48.1.bcr.1", repo_name = "com_github_grpc_grpc")
bazel_dep(name = "grpc", version = "1.66.0.bcr.2", repo_name = "com_github_grpc_grpc")
bazel_dep(name = "grpc-java", version = "1.66.0")
bazel_dep(name = "googleapis", version = "0.0.0-20240819-fe8ba054a")
bazel_dep(name = "platforms", version = "0.0.10")
bazel_dep(name = "rules_pkg", version = "0.9.1")
bazel_dep(name = "rules_pkg", version = "1.0.1")
bazel_dep(name = "stardoc", version = "0.7.1", repo_name = "io_bazel_skydoc")
bazel_dep(name = "zstd-jni", version = "1.5.2-3.bcr.1")
bazel_dep(name = "blake3", version = "1.5.1.bcr.1")
Expand All @@ -29,13 +30,37 @@ bazel_dep(name = "rules_proto", version = "6.0.2")
bazel_dep(name = "rules_jvm_external", version = "6.0")
bazel_dep(name = "rules_python", version = "0.36.0")
bazel_dep(name = "rules_testing", version = "0.6.0")
bazel_dep(name = "googletest", version = "1.14.0.bcr.1", repo_name = "com_google_googletest")
bazel_dep(name = "googletest", version = "1.15.2", repo_name = "com_google_googletest")
bazel_dep(name = "with_cfg.bzl", version = "0.2.4")
bazel_dep(name = "abseil-cpp", version = "20230802.1")
bazel_dep(name = "abseil-cpp", version = "20240722.0")

# TODO(pcloudy): Add remoteapis and googleapis as Bazel modules in the BCR.
# TODO: wyv@ - use release 29.0 when that's out
bazel_dep(name = "protobuf", version = "29.0-dev", repo_name = "com_google_protobuf")
archive_override(
module_name = "protobuf",
integrity = "sha256-zF1Z3SMnHqcP1QKIeAoGGZDEARNXRWRgZi70eKldVlc=",
patch_strip = 1,
# Temporarily patch out rules_rust stuff from protobuf. Not just because we don't need it,
# but also because it introduces huge dependency bloat: rules_rust -> aspect_rules_js ->
# aspect_rules_lint -> rules_buf.
patches = ["//third_party/protobuf:remove_rules_rust.patch"],
strip_prefix = "protobuf-3b62052186d39775090fb074adcba078ea622f54",
urls = ["https://github.com/protocolbuffers/protobuf/archive/3b62052186d39775090fb074adcba078ea622f54.zip"],
)

# TODO: wyv@ - add remoteapis to the BCR
bazel_dep(name = "remoteapis", version = "")
bazel_dep(name = "googleapis", version = "")

# TODO: wyv@ - eventually this part should be unnecessary (after we fix up the googleapis module
# in the BCR)
switched_rules = use_extension("@googleapis//:extensions.bzl", "switched_rules")
switched_rules.use_languages(
cc = True,
go = True,
grpc = True,
java = True,
python = True,
)

single_version_override(
module_name = "rules_jvm_external",
Expand All @@ -49,41 +74,12 @@ local_path_override(
path = "./third_party/remoteapis",
)

local_path_override(
module_name = "googleapis",
path = "./third_party/googleapis",
)

single_version_override(
module_name = "grpc",
patch_strip = 1,
patches = [
"//third_party/grpc:00_disable_layering_check.patch",
],
)

single_version_override(
module_name = "protobuf",
version = "21.7",
patch_strip = 1,
patches = ["//third_party:protobuf_21.7.patch"],
)

# The following Bazel modules are not direct dependencies for building Bazel,
# but are required for visibility from DIST_ARCHIVE_REPOS in repositories.bzl
bazel_dep(name = "apple_support", version = "1.8.1")
bazel_dep(name = "apple_support", version = "1.15.1")
bazel_dep(name = "c-ares", version = "1.15.0")
bazel_dep(name = "rules_go", version = "0.39.1")
bazel_dep(name = "rules_go", version = "0.48.0")
bazel_dep(name = "rules_kotlin", version = "1.9.6")
bazel_dep(name = "upb", version = "0.0.0-20220923-a547704")
single_version_override(
module_name = "upb",
patch_strip = 1,
patches = [
"//third_party/upb:00_remove_toolchain_transition.patch",
"//third_party/upb:01_remove_werror.patch",
],
)

# =========================================
# Java dependencies
Expand Down Expand Up @@ -117,7 +113,7 @@ maven.install(
"com.google.flogger:flogger-system-backend:0.5.1",
"com.google.flogger:google-extensions:0.5.1",
"com.google.guava:failureaccess:1.0.1",
"com.google.guava:guava:33.0.0-jre",
"com.google.guava:guava:33.2.1-jre",
"com.google.http-client:google-http-client:1.42.0",
"com.google.http-client:google-http-client-gson:1.42.0",
"com.google.j2objc:j2objc-annotations:1.3",
Expand All @@ -130,41 +126,42 @@ maven.install(
"commons-collections:commons-collections:3.2.2",
"commons-lang:commons-lang:2.6",
"io.github.java-diff-utils:java-diff-utils:4.12",
"io.grpc:grpc-api:1.48.1",
"io.grpc:grpc-auth:1.48.1",
"io.grpc:grpc-context:1.48.1",
"io.grpc:grpc-core:1.48.1",
"io.grpc:grpc-netty:1.48.1",
"io.grpc:grpc-protobuf:1.48.1",
"io.grpc:grpc-protobuf-lite:1.48.1",
"io.grpc:grpc-stub:1.48.1",
"io.netty:netty-buffer:4.1.93.Final",
"io.netty:netty-codec:4.1.93.Final",
"io.netty:netty-codec-http:4.1.93.Final",
"io.netty:netty-codec-http2:4.1.93.Final",
"io.netty:netty-common:4.1.93.Final",
"io.netty:netty-handler:4.1.93.Final",
"io.netty:netty-handler-proxy:4.1.93.Final",
"io.netty:netty-resolver:4.1.93.Final",
"io.netty:netty-resolver-dns:4.1.93.Final",
"io.grpc:grpc-api:1.66.0",
"io.grpc:grpc-auth:1.66.0",
"io.grpc:grpc-context:1.66.0",
"io.grpc:grpc-core:1.66.0",
"io.grpc:grpc-inprocess:1.66.0",
"io.grpc:grpc-netty:1.66.0",
"io.grpc:grpc-protobuf:1.66.0",
"io.grpc:grpc-protobuf-lite:1.66.0",
"io.grpc:grpc-stub:1.66.0",
"io.netty:netty-buffer:4.1.100.Final",
"io.netty:netty-codec:4.1.100.Final",
"io.netty:netty-codec-http:4.1.100.Final",
"io.netty:netty-codec-http2:4.1.100.Final",
"io.netty:netty-common:4.1.100.Final",
"io.netty:netty-handler:4.1.100.Final",
"io.netty:netty-handler-proxy:4.1.100.Final",
"io.netty:netty-resolver:4.1.100.Final",
"io.netty:netty-resolver-dns:4.1.100.Final",
"io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64:2.0.56.Final",
"io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64:2.0.56.Final",
"io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64:2.0.56.Final",
"io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64:2.0.56.Final",
"io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64:2.0.56.Final",
"io.netty:netty-tcnative-classes:2.0.56.Final",
"io.netty:netty-transport:4.1.93.Final",
"io.netty:netty-transport-classes-epoll:4.1.93.Final",
"io.netty:netty-transport-classes-kqueue:4.1.93.Final",
"io.netty:netty-transport-native-epoll:jar:linux-aarch_64:4.1.93.Final",
"io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.93.Final",
"io.netty:netty-transport-native-kqueue:jar:osx-aarch_64:4.1.93.Final",
"io.netty:netty-transport-native-kqueue:jar:osx-x86_64:4.1.93.Final",
"io.netty:netty-transport-native-unix-common:4.1.93.Final",
"io.netty:netty-transport-native-unix-common:jar:linux-aarch_64:4.1.93.Final",
"io.netty:netty-transport-native-unix-common:jar:linux-x86_64:4.1.93.Final",
"io.netty:netty-transport-native-unix-common:jar:osx-aarch_64:4.1.93.Final",
"io.netty:netty-transport-native-unix-common:jar:osx-x86_64:4.1.93.Final",
"io.netty:netty-transport:4.1.100.Final",
"io.netty:netty-transport-classes-epoll:4.1.100.Final",
"io.netty:netty-transport-classes-kqueue:4.1.100.Final",
"io.netty:netty-transport-native-epoll:jar:linux-aarch_64:4.1.100.Final",
"io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.100.Final",
"io.netty:netty-transport-native-kqueue:jar:osx-aarch_64:4.1.100.Final",
"io.netty:netty-transport-native-kqueue:jar:osx-x86_64:4.1.100.Final",
"io.netty:netty-transport-native-unix-common:4.1.100.Final",
"io.netty:netty-transport-native-unix-common:jar:linux-aarch_64:4.1.100.Final",
"io.netty:netty-transport-native-unix-common:jar:linux-x86_64:4.1.100.Final",
"io.netty:netty-transport-native-unix-common:jar:osx-aarch_64:4.1.100.Final",
"io.netty:netty-transport-native-unix-common:jar:osx-x86_64:4.1.100.Final",
"io.reactivex.rxjava3:rxjava:3.1.2",
"it.unimi.dsi:fastutil:7.2.1",
"javax.activation:javax.activation-api:1.2.0",
Expand Down Expand Up @@ -199,11 +196,7 @@ maven.install(
"org.apache.httpcomponents:httpclient",
"org.apache.httpcomponents:httpcore",
"org.eclipse.jgit:org.eclipse.jgit",
# We build protobuf Java library from source, exclude protobuf jars to be safe.
"com.google.protobuf:protobuf-java",
"com.google.protobuf:protobuf-javalite",
],
# Don't forget to change this to back to True before submitting your change.
fail_if_repin_required = True,
lock_file = "//:maven_install.json",
repositories = [
Expand All @@ -221,7 +214,7 @@ maven.install(
version = version,
)
for group, artifact, version in [coord.split(":") for coord in [
"com.google.guava:guava-testlib:33.0.0-jre",
"com.google.guava:guava-testlib:33.2.1-jre",
"com.google.jimfs:jimfs:1.2",
"com.google.testing.compile:compile-testing:0.18",
"com.google.testparameterinjector:test-parameter-injector:1.16",
Expand All @@ -233,7 +226,7 @@ maven.install(
]]
]

use_repo(maven, "maven", "unpinned_maven")
use_repo(maven, "maven")

java_toolchains = use_extension("@rules_java//java:extensions.bzl", "toolchains")
use_repo(
Expand Down
Loading

0 comments on commit c46b9ce

Please sign in to comment.