diff --git a/cloud_gcp/BUILD.bazel b/cloud_gcp/BUILD.bazel new file mode 100644 index 0000000000..7e7dccf963 --- /dev/null +++ b/cloud_gcp/BUILD.bazel @@ -0,0 +1,91 @@ +scala_library( + name = "lib", + srcs = glob(["src/main/**/*.scala"]), + visibility = ["//visibility:public"], + deps = [ + "//api:thrift", + "//api:lib", + "//online:lib", + "//spark:lib", + "//tools/build_rules/spark:spark-exec", + scala_artifact("org.scala-lang.modules:scala-java8-compat"), + scala_artifact("org.json4s:json4s-core"), + scala_artifact("org.json4s:json4s-jackson"), + scala_artifact("org.json4s:json4s-ast"), + scala_artifact("org.scala-lang.modules:scala-collection-compat"), + scala_artifact("org.rogach:scallop"), + maven_artifact("com.google.cloud:google-cloud-core"), + maven_artifact("com.google.cloud:google-cloud-bigquery"), + maven_artifact("com.google.cloud:google-cloud-bigtable"), + maven_artifact("com.google.cloud:google-cloud-pubsub"), + maven_artifact("com.google.cloud:google-cloud-dataproc"), + maven_artifact("com.google.cloud.bigdataoss:gcsio"), + maven_artifact("com.google.cloud.bigdataoss:util-hadoop"), + scala_artifact("com.google.cloud.spark:spark-bigquery-with-dependencies"), + maven_artifact("com.google.api:api-common"), + maven_artifact("com.google.api.grpc:proto-google-cloud-dataproc-v1"), + maven_artifact("com.google.api:gax"), + maven_artifact("com.google.guava:guava"), + maven_artifact("com.google.protobuf:protobuf-java"), + maven_artifact("org.yaml:snakeyaml"), + maven_artifact("io.grpc:grpc-netty-shaded"), + maven_artifact("ch.qos.reload4j:reload4j"), + maven_artifact("org.slf4j:slf4j-api"), + maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"), + maven_artifact("org.threeten:threetenbp"), + ], +) + +test_deps = [ + ":lib", + "//api:thrift", + "//api:lib", + "//online:lib", + "//spark:lib", + "//tools/build_rules/spark:spark-exec", + # Libraries + scala_artifact("org.scala-lang.modules:scala-java8-compat"), + scala_artifact("org.scala-lang.modules:scala-collection-compat"), + maven_artifact("com.google.cloud:google-cloud-bigquery"), + maven_artifact("com.google.cloud:google-cloud-bigtable"), + maven_artifact("com.google.cloud:google-cloud-dataproc"), + maven_artifact("com.google.cloud.bigdataoss:gcs-connector"), + maven_artifact("com.google.cloud.bigdataoss:gcsio"), + maven_artifact("com.google.cloud.bigdataoss:util-hadoop"), + maven_artifact("com.google.cloud:google-cloud-bigtable-emulator"), + maven_artifact("com.google.api:api-common"), + maven_artifact("com.google.api.grpc:proto-google-cloud-dataproc-v1"), + scala_artifact("com.google.cloud.spark:spark-bigquery-with-dependencies"), + maven_artifact("com.google.api:gax"), + maven_artifact("com.google.protobuf:protobuf-java"), + # Testing + scala_artifact("org.scalatest:scalatest-matchers-core"), + scala_artifact("org.scalatest:scalatest-core"), + scala_artifact("org.scalatest:scalatest"), + scala_artifact("org.scalatest:scalatest-flatspec"), + scala_artifact("org.scalatest:scalatest-funsuite"), + scala_artifact("org.scalatest:scalatest-shouldmatchers"), + scala_artifact("org.scalactic:scalactic"), + scala_artifact("org.scalatestplus:mockito-3-4"), + scala_artifact("org.mockito:mockito-scala"), + maven_artifact("org.mockito:mockito-core"), + maven_artifact("org.scalatest:scalatest-compatible"), + maven_artifact("junit:junit"), + maven_artifact("com.novocode:junit-interface"), +] + +scala_library( + name = "test-lib", + srcs = glob(["src/test/**/*.scala"]), + visibility = ["//visibility:public"], + deps = test_deps, +) + +scala_test_suite( + name = "test", + srcs = glob(["src/test/**/*.scala"]), + visibility = ["//visibility:public"], + # defined in prelude_bazel file + jvm_flags = _JVM_FLAGS_FOR_ACCESSING_BASE_JAVA_CLASSES, + deps = test_deps + [":test-lib"], +) \ No newline at end of file diff --git a/flink/BUILD.bazel b/flink/BUILD.bazel index 52a803cd49..445f62ceb6 100644 --- a/flink/BUILD.bazel +++ b/flink/BUILD.bazel @@ -41,6 +41,7 @@ scala_library( test_deps = flink_test_deps + [ ":lib", "//online:lib", + "//online:test-lib", "//api:thrift", "//api:lib", "//aggregator:lib", diff --git a/online/BUILD.bazel b/online/BUILD.bazel index 3767a20488..ded5473d7d 100644 --- a/online/BUILD.bazel +++ b/online/BUILD.bazel @@ -27,57 +27,43 @@ scala_library( ], ) +test_deps = [ + ":lib", + "//api:thrift", + "//api:lib", + "//aggregator:lib", + "//tools/build_rules/spark:spark-exec", + # Libraries + maven_artifact("com.github.ben-manes.caffeine:caffeine"), + maven_artifact("org.slf4j:slf4j-api"), + maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"), + maven_artifact("com.google.code.gson:gson"), + # Testing + scala_artifact("org.scalatest:scalatest-matchers-core"), + scala_artifact("org.scalatest:scalatest-core"), + scala_artifact("org.scalatest:scalatest"), + scala_artifact("org.scalatest:scalatest-flatspec"), + scala_artifact("org.scalatest:scalatest-funsuite"), + scala_artifact("org.scalatest:scalatest-shouldmatchers"), + scala_artifact("org.scalactic:scalactic"), + scala_artifact("org.scalatestplus:mockito-3-4"), + scala_artifact("org.mockito:mockito-scala"), + maven_artifact("org.mockito:mockito-core"), + maven_artifact("org.scalatest:scalatest-compatible"), + maven_artifact("junit:junit"), + maven_artifact("com.novocode:junit-interface"), +] + scala_library( name = "test-lib", srcs = glob(["src/test/**/*.scala"]), visibility = ["//visibility:public"], - deps = [ - ":lib", - "//api:thrift", - "//api:lib", - "//aggregator:lib", - "//tools/build_rules/spark:spark-exec", - # Libraries - maven_artifact("com.github.ben-manes.caffeine:caffeine"), - maven_artifact("org.slf4j:slf4j-api"), - maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"), - maven_artifact("com.google.code.gson:gson"), - # Testing - scala_artifact("org.scalatest:scalatest-matchers-core"), - scala_artifact("org.scalatest:scalatest-core"), - scala_artifact("org.scalatest:scalatest"), - scala_artifact("org.scalatest:scalatest-flatspec"), - scala_artifact("org.scalatest:scalatest-funsuite"), - scala_artifact("org.scalatest:scalatest-shouldmatchers"), - scala_artifact("org.scalactic:scalactic"), - scala_artifact("org.scalatestplus:mockito-3-4"), - scala_artifact("org.mockito:mockito-scala"), - maven_artifact("org.mockito:mockito-core"), - maven_artifact("org.scalatest:scalatest-compatible"), - maven_artifact("junit:junit"), - maven_artifact("com.novocode:junit-interface"), - ], + deps = test_deps, ) scala_test_suite( name = "test", srcs = glob(["src/test/**/*.scala"]), visibility = ["//visibility:public"], - deps = [ - ":lib", - ":test-lib", - "//aggregator:lib", - "//api:lib", - "//api:thrift", - "//tools/build_rules/spark:spark-exec", - scala_artifact("org.scalatestplus:mockito-3-4"), - maven_artifact("org.mockito:mockito-core"), - scala_artifact("org.scala-lang.modules:scala-collection-compat"), - maven_artifact("org.slf4j:slf4j-api"), - maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"), - maven_artifact("com.google.code.gson:gson"), - maven_artifact("com.github.ben-manes.caffeine:caffeine"), - maven_artifact("junit:junit"), - maven_artifact("com.novocode:junit-interface"), - ], + deps = test_deps + [":test-lib"], ) \ No newline at end of file diff --git a/online/src/test/scala/ai/chronon/online/test/FetcherBaseTest.scala b/online/src/test/scala/ai/chronon/online/test/FetcherBaseTest.scala index 09c30e8f73..c79a99bc6a 100644 --- a/online/src/test/scala/ai/chronon/online/test/FetcherBaseTest.scala +++ b/online/src/test/scala/ai/chronon/online/test/FetcherBaseTest.scala @@ -64,7 +64,7 @@ class FetcherBaseTest extends AnyFlatSpec with MockitoSugar with Matchers with M // Future compositions in the Fetcher so provision it in // the mock to prevent hanging. when(kvStore.executionContext).thenReturn(ExecutionContext.global) - fetcherBase = spy(new FetcherBase(kvStore)) + fetcherBase = spy[FetcherBase](new FetcherBase(kvStore)) } it should "fetch columns single query" in { @@ -213,7 +213,7 @@ class FetcherBaseTest extends AnyFlatSpec with MockitoSugar with Matchers with M kvStore = mock[KVStore](Answers.RETURNS_DEEP_STUBS) when(kvStore.executionContext).thenReturn(ExecutionContext.global) - val fetcherBaseWithFlagStore = spy(new FetcherBase(kvStore, flagStore = flagStore)) + val fetcherBaseWithFlagStore = spy[FetcherBase](new FetcherBase(kvStore, flagStore = flagStore)) when(fetcherBaseWithFlagStore.isCacheSizeConfigured).thenReturn(true) def buildGroupByWithCustomJson(name: String): GroupBy = Builders.GroupBy(metaData = Builders.MetaData(name = name)) diff --git a/online/src/test/scala/ai/chronon/online/test/FetcherCacheTest.scala b/online/src/test/scala/ai/chronon/online/test/FetcherCacheTest.scala index 84dc46bcba..c4e3923048 100644 --- a/online/src/test/scala/ai/chronon/online/test/FetcherCacheTest.scala +++ b/online/src/test/scala/ai/chronon/online/test/FetcherCacheTest.scala @@ -141,7 +141,7 @@ class FetcherCacheTest extends AnyFlatSpec with MockitoHelper { it should "get cached requests does not cache when cache is disabled for group by" in { val testCache = new BatchIrCache("test", batchIrCacheMaximumSize) - val spiedTestCache = spy(testCache) + val spiedTestCache = spy[BatchIrCache](testCache) val fetcherCache = new TestableFetcherCache(Some(testCache)) { // Cache is enabled globally, but disabled for a specific groupBy override def isCachingEnabled(groupBy: GroupBy) = false @@ -216,7 +216,7 @@ class FetcherCacheTest extends AnyFlatSpec with MockitoHelper { val cacheKey = BatchIrCache.Key(servingInfo.groupByOps.batchDataset, keys, servingInfo.batchEndTsMillis) val fetcherCache = new TestableFetcherCache(Some(batchIrCache)) - val spiedFetcherCache = Mockito.spy(fetcherCache) + val spiedFetcherCache = Mockito.spy[TestableFetcherCache](fetcherCache) doReturn(true).when(spiedFetcherCache).isCachingEnabled(any()) // 1. Cached BatchResponse returns the same IRs passed in @@ -247,7 +247,7 @@ class FetcherCacheTest extends AnyFlatSpec with MockitoHelper { val toBatchIr = mock[(Array[Byte], GroupByServingInfoParsed) => FinalBatchIr] val kvStoreBatchResponses = BatchResponses(Success(Seq(TimedValue(batchBytes, 1000L)))) - val spiedFetcherCache = Mockito.spy(new TestableFetcherCache(None)) + val spiedFetcherCache = Mockito.spy[TestableFetcherCache](new TestableFetcherCache(None)) when(toBatchIr(any(), any())).thenReturn(finalBatchIr) // When getBatchIrFromBatchResponse is called, it decodes the bytes and doesn't hit the cache @@ -272,7 +272,7 @@ class FetcherCacheTest extends AnyFlatSpec with MockitoHelper { val keys = Map("key" -> "value") val cacheKey = BatchIrCache.Key(servingInfo.groupByOps.batchDataset, keys, servingInfo.batchEndTsMillis) - val spiedFetcherCache = Mockito.spy(new TestableFetcherCache(Some(batchIrCache))) + val spiedFetcherCache = Mockito.spy[TestableFetcherCache](new TestableFetcherCache(Some(batchIrCache))) doReturn(true).when(spiedFetcherCache).isCachingEnabled(any()) // 1. Cached BatchResponse returns the same Map responses passed in @@ -313,7 +313,7 @@ class FetcherCacheTest extends AnyFlatSpec with MockitoHelper { when(servingInfo.outputCodec).thenReturn(outputCodec) when(outputCodec.decodeMap(any())).thenReturn(mapResponse) - val spiedFetcherCache = Mockito.spy(new TestableFetcherCache(None)) + val spiedFetcherCache = Mockito.spy[TestableFetcherCache](new TestableFetcherCache(None)) // When getMapResponseFromBatchResponse is called, it decodes the bytes and doesn't hit the cache val decodedMapResponse = spiedFetcherCache.getMapResponseFromBatchResponse(kvStoreBatchResponses, diff --git a/service_commons/BUILD.bazel b/service_commons/BUILD.bazel index 481f9552f8..5296f83aa2 100644 --- a/service_commons/BUILD.bazel +++ b/service_commons/BUILD.bazel @@ -2,7 +2,7 @@ java_library( name = "lib", srcs = glob(["src/main/**/*.java"]), visibility = ["//visibility:public"], - deps = SCALA_DEPS + [ + deps = _SCALA_DEPS + [ "//api:lib", "//online:lib", maven_artifact("io.vertx:vertx-core"), @@ -20,23 +20,27 @@ java_library( ], ) +test_deps = [ + ":lib", + "//api:lib", + "//api:thrift", + "//online:lib", + # Libraries + maven_artifact("io.vertx:vertx-core"), + maven_artifact("io.vertx:vertx-web"), + maven_artifact("io.vertx:vertx-web-client"), + # Testing + maven_artifact("io.vertx:vertx-junit5"), + maven_artifact("org.junit.jupiter:junit-jupiter-api"), + maven_artifact("org.junit.platform:junit-platform-launcher"), + maven_artifact("org.junit.platform:junit-platform-reporting"), +] + java_library( name = "test-lib", srcs = glob(["src/test/**/*.java"]), visibility = ["//visibility:public"], - deps = [ - ":lib", - "//api:lib", - "//api:thrift", - "//online:lib", - # Libraries - maven_artifact("io.vertx:vertx-core"), - maven_artifact("io.vertx:vertx-web"), - maven_artifact("io.vertx:vertx-web-client"), - # Testing - maven_artifact("io.vertx:vertx-junit5"), - maven_artifact("org.junit.jupiter:junit-jupiter-api"), - ], + deps = test_deps, ) java_test_suite( @@ -44,20 +48,5 @@ java_test_suite( srcs = glob(["src/test/**/*.java"]), visibility = ["//visibility:public"], runner = "junit5", - deps = [ - ":lib", - ":test-lib", - "//api:lib", - "//api:thrift", - "//online:lib", - # Libraries - maven_artifact("io.vertx:vertx-core"), - maven_artifact("io.vertx:vertx-web"), - maven_artifact("io.vertx:vertx-web-client"), - # Testing - maven_artifact("io.vertx:vertx-junit5"), - maven_artifact("org.junit.jupiter:junit-jupiter-api"), - maven_artifact("org.junit.platform:junit-platform-launcher"), - maven_artifact("org.junit.platform:junit-platform-reporting"), - ], + deps = test_deps + [":test-lib"], ) \ No newline at end of file diff --git a/spark/BUILD.bazel b/spark/BUILD.bazel index 95acc8e745..52a50e43ed 100644 --- a/spark/BUILD.bazel +++ b/spark/BUILD.bazel @@ -1,9 +1,7 @@ scala_library( name = "lib", srcs = glob(["src/main/**/*.scala"]), - resources = [ - "src/main/resources/log4j2.properties", - ], + visibility = ["//visibility:public"], deps = [ "//aggregator:lib", "//api:lib", @@ -23,6 +21,8 @@ scala_library( scala_artifact("org.scala-lang.modules:scala-collection-compat"), scala_artifact("org.scala-lang.modules:scala-parser-combinators"), maven_artifact("org.slf4j:slf4j-api"), + maven_artifact("org.apache.logging.log4j:log4j-api"), + maven_artifact("org.apache.logging.log4j:log4j-core"), maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"), maven_artifact("com.google.code.gson:gson"), maven_artifact("jakarta.servlet:jakarta.servlet-api"), @@ -38,107 +38,61 @@ scala_library( maven_artifact("io.netty:netty-common"), maven_artifact("io.netty:netty-codec"), scala_artifact("io.delta:delta-spark"), + maven_artifact("net.sf.py4j:py4j"), ], ) +test_deps = [ + ":lib", + "//aggregator:lib", + "//aggregator:test-lib", + "//api:lib", + "//api:thrift", + "//online:lib", + "//tools/build_rules/spark:spark-exec", + # Library + maven_artifact("com.google.code.gson:gson"), + maven_artifact("com.google.guava:guava"), + scala_artifact("org.rogach:scallop"), + maven_artifact("commons.io:commons-io"), + maven_artifact("commons.lang:commons-lang"), + scala_artifact("org.scala-lang.modules:scala-java8-compat"), + scala_artifact("com.fasterxml.jackson.module:jackson-module-scala"), + maven_artifact("org.slf4j:slf4j-api"), + maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"), + scala_artifact("org.json4s:json4s-core"), + scala_artifact("org.json4s:json4s-jackson"), + scala_artifact("org.json4s:json4s-ast"), + maven_artifact("org.yaml:snakeyaml"), + # Testing + scala_artifact("org.scalatest:scalatest-matchers-core"), + scala_artifact("org.scalatest:scalatest-core"), + scala_artifact("org.scalatest:scalatest"), + scala_artifact("org.scalatest:scalatest-flatspec"), + scala_artifact("org.scalatest:scalatest-funsuite"), + scala_artifact("org.scalatest:scalatest-shouldmatchers"), + scala_artifact("org.scalactic:scalactic"), + scala_artifact("org.scalatestplus:mockito-3-4"), + maven_artifact("org.scalatest:scalatest-compatible"), + maven_artifact("junit:junit"), + maven_artifact("com.novocode:junit-interface"), + maven_artifact("org.mockito:mockito-core"), +] + scala_library( name = "test-lib", srcs = glob(["src/test/**/*.scala"]), visibility = ["//visibility:public"], - deps = [ - ":lib", - "//aggregator:lib", - "//aggregator:test-lib", - "//api:lib", - "//api:thrift", - "//online:lib", - "//tools/build_rules/spark:spark-exec", - # Library - maven_artifact("com.google.code.gson:gson"), - maven_artifact("com.google.guava:guava"), - scala_artifact("org.rogach:scallop"), - maven_artifact("commons.io:commons-io"), - maven_artifact("commons.lang:commons-lang"), - scala_artifact("org.scala-lang.modules:scala-java8-compat"), - scala_artifact("com.fasterxml.jackson.module:jackson-module-scala"), - maven_artifact("org.slf4j:slf4j-api"), - maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"), - scala_artifact("org.json4s:json4s-core"), - scala_artifact("org.json4s:json4s-jackson"), - scala_artifact("org.json4s:json4s-ast"), - maven_artifact("org.yaml:snakeyaml"), - # Testing - scala_artifact("org.scalatest:scalatest-matchers-core"), - scala_artifact("org.scalatest:scalatest-core"), - scala_artifact("org.scalatest:scalatest"), - scala_artifact("org.scalatest:scalatest-flatspec"), - scala_artifact("org.scalatest:scalatest-funsuite"), - scala_artifact("org.scalatest:scalatest-shouldmatchers"), - scala_artifact("org.scalactic:scalactic"), - scala_artifact("org.scalatestplus:mockito-3-4"), - maven_artifact("org.scalatest:scalatest-compatible"), - maven_artifact("junit:junit"), - maven_artifact("com.novocode:junit-interface"), - maven_artifact("org.mockito:mockito-core"), - ], + deps = test_deps, ) scala_test_suite( name = "test", srcs = glob(["src/test/**/*.scala"]), visibility = ["//visibility:public"], - jvm_flags = [ - "--add-opens=java.base/java.lang=ALL-UNNAMED", - "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", - "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", - "--add-opens=java.base/java.io=ALL-UNNAMED", - "--add-opens=java.base/java.net=ALL-UNNAMED", - "--add-opens=java.base/java.nio=ALL-UNNAMED", - "--add-opens=java.base/java.util=ALL-UNNAMED", - "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", - "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", - "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", - "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", - "--add-opens=java.base/sun.security.action=ALL-UNNAMED", - "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", - ], - deps = [ - ":lib", - ":test-lib", - "//aggregator:lib", - "//aggregator:test-lib", - "//api:lib", - "//api:thrift", - "//online:lib", - "//tools/build_rules/spark:spark-exec", - # Library - maven_artifact("com.google.code.gson:gson"), - maven_artifact("com.google.guava:guava"), - scala_artifact("org.rogach:scallop"), - maven_artifact("commons.io:commons-io"), - maven_artifact("commons.lang:commons-lang"), - scala_artifact("org.scala-lang.modules:scala-java8-compat"), - scala_artifact("com.fasterxml.jackson.module:jackson-module-scala"), - maven_artifact("org.slf4j:slf4j-api"), - maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"), - scala_artifact("org.json4s:json4s-core"), - scala_artifact("org.json4s:json4s-jackson"), - scala_artifact("org.json4s:json4s-ast"), - maven_artifact("org.yaml:snakeyaml"), - # Testing - scala_artifact("org.scalatest:scalatest-matchers-core"), - scala_artifact("org.scalatest:scalatest-core"), - scala_artifact("org.scalatest:scalatest"), - scala_artifact("org.scalatest:scalatest-flatspec"), - scala_artifact("org.scalatest:scalatest-funsuite"), - scala_artifact("org.scalatest:scalatest-shouldmatchers"), - scala_artifact("org.scalactic:scalactic"), - scala_artifact("org.scalatestplus:mockito-3-4"), - maven_artifact("org.scalatest:scalatest-compatible"), - maven_artifact("junit:junit"), - maven_artifact("com.novocode:junit-interface"), - maven_artifact("org.mockito:mockito-core"), - ], + # defined in prelude_bazel file + jvm_flags = _JVM_FLAGS_FOR_ACCESSING_BASE_JAVA_CLASSES, + deps = test_deps + [":test-lib"], ) jvm_binary( diff --git a/tools/build_rules/dependencies/maven_repository.bzl b/tools/build_rules/dependencies/maven_repository.bzl index 0735bfe616..62d18a9e61 100644 --- a/tools/build_rules/dependencies/maven_repository.bzl +++ b/tools/build_rules/dependencies/maven_repository.bzl @@ -18,7 +18,7 @@ maven_repository = repository( "org.junit.platform:junit-platform-reporting:1.10.5", "com.novocode:junit-interface:0.11", "org.scalatestplus:mockito-3-4_2.12:3.2.10.0", - "org.mockito:mockito-core:4.6.1", + "org.mockito:mockito-core:5.12.0", "org.mockito:mockito-scala_2.12:1.17.0", "org.scalatest:scalatest_2.12:3.2.15", "org.scalatest:scalatest-shouldmatchers_2.12:3.2.15", @@ -50,13 +50,29 @@ maven_repository = repository( "commons-lang:commons-lang:2.6", "io.netty:netty-all:4.1.111.Final", "io.delta:delta-spark_2.12:3.2.0", + "io.grpc:grpc-netty-shaded:1.62.2", + "ch.qos.reload4j:reload4j:1.2.25", "ch.qos.logback:logback-classic:1.5.6", "com.typesafe:config:1.4.3", "io.micrometer:micrometer-registry-statsd:1.13.6", + "net.sf.py4j:py4j:0.10.9.9", # Kafka "org.apache.kafka:kafka-clients:3.9.0", + # Google Cloud + "com.google.cloud:google-cloud-bigquery:2.42.0", + "com.google.cloud:google-cloud-bigtable:2.41.0", + "com.google.cloud:google-cloud-pubsub:1.131.0", + "com.google.cloud:google-cloud-dataproc:4.52.0", + # Have to specify in group:artifact:packaging:version format if version doesn't start with a digit + # Code reference: https://github.com/bazel-contrib/rules_jvm_external/blob/master/private/lib/coordinates.bzl#L44 + "com.google.cloud.bigdataoss:gcs-connector:jar:hadoop3-2.2.6", + "com.google.cloud.bigdataoss:gcsio:3.0.3", + "com.google.cloud.bigdataoss:util-hadoop:3.0.0", + "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.41.0", + "com.google.cloud:google-cloud-bigtable-emulator:0.178.0", + # Flink "org.apache.flink:flink-streaming-scala_2.12:1.17.0", "org.apache.flink:flink-metrics-dropwizard:1.17.0", diff --git a/tools/build_rules/dependencies/spark_repository.bzl b/tools/build_rules/dependencies/spark_repository.bzl index e6dcafaff6..dd9df739b8 100644 --- a/tools/build_rules/dependencies/spark_repository.bzl +++ b/tools/build_rules/dependencies/spark_repository.bzl @@ -13,6 +13,8 @@ spark_repository = repository( # Hive dependencies "org.apache.hive:hive-metastore:2.3.9", "org.apache.hive:hive-exec:2.3.9", + + "org.apache.curator:apache-curator:5.5.0", ], excluded_artifacts = [ "org.pentaho:pentaho-aggdesigner-algorithm", diff --git a/tools/build_rules/prelude_bazel b/tools/build_rules/prelude_bazel index ae397b59c3..0aa82244fc 100644 --- a/tools/build_rules/prelude_bazel +++ b/tools/build_rules/prelude_bazel @@ -8,7 +8,23 @@ load("@contrib_rules_jvm//docs:stardoc-input.bzl", "java_test_suite") load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library", "scala_test_suite") load("//tools/build_rules:maven_artifact.bzl", "maven_artifact", "scala_artifact") -SCALA_DEPS = [ +_SCALA_DEPS = [ "@io_bazel_rules_scala_scala_library_2_12_18//:io_bazel_rules_scala_scala_library_2_12_18", # other scala related deps... +] + +_JVM_FLAGS_FOR_ACCESSING_BASE_JAVA_CLASSES = [ + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", ] \ No newline at end of file