From cd1378320fb98c6e55694783e5654dc6147938ae Mon Sep 17 00:00:00 2001 From: tchow-zlai Date: Fri, 31 Jan 2025 18:21:18 -0800 Subject: [PATCH 1/2] feat: upgrade artifact upload to use bazel Co-authored-by: Thomas Chow Co-authored-by: Thomas Chow --- api/py/ai/chronon/repo/run.py | 4 ++-- cloud_gcp/BUILD.bazel | 7 +++++++ distribution/build_and_upload_gcp_artifacts.sh | 9 +++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/api/py/ai/chronon/repo/run.py b/api/py/ai/chronon/repo/run.py index bc1b38c1c8..ecbe7d4a53 100755 --- a/api/py/ai/chronon/repo/run.py +++ b/api/py/ai/chronon/repo/run.py @@ -130,10 +130,10 @@ # GCP DATAPROC SPECIFIC CONSTANTS DATAPROC_ENTRY = "ai.chronon.integrations.cloud_gcp.DataprocSubmitter" -ZIPLINE_ONLINE_JAR_DEFAULT = "cloud_gcp-assembly-0.1.0-SNAPSHOT.jar" +ZIPLINE_ONLINE_JAR_DEFAULT = "lib_deploy.jar" ZIPLINE_ONLINE_CLASS_DEFAULT = "ai.chronon.integrations.cloud_gcp.GcpApiImpl" ZIPLINE_FLINK_JAR_DEFAULT = "flink-assembly-0.1.0-SNAPSHOT.jar" -ZIPLINE_DATAPROC_SUBMITTER_JAR = "cloud_gcp_submitter-assembly-0.1.0-SNAPSHOT.jar" +ZIPLINE_DATAPROC_SUBMITTER_JAR = "submitter_deploy.jar" class DataprocJobType(Enum): diff --git a/cloud_gcp/BUILD.bazel b/cloud_gcp/BUILD.bazel index f32705b438..09b3b44946 100644 --- a/cloud_gcp/BUILD.bazel +++ b/cloud_gcp/BUILD.bazel @@ -36,6 +36,12 @@ scala_library( ], ) +jvm_binary( + name = "submitter", + main_class = "ai.chronon.integrations.cloud_gcp.DataprocSubmitter", + runtime_deps = [":lib"], +) + test_deps = [ ":lib", "//api:thrift", @@ -82,6 +88,7 @@ scala_library( deps = test_deps, ) + scala_test_suite( name = "test", srcs = glob(["src/test/**/*.scala"]), diff --git a/distribution/build_and_upload_gcp_artifacts.sh b/distribution/build_and_upload_gcp_artifacts.sh index b42517f73c..9e27a86991 100755 --- a/distribution/build_and_upload_gcp_artifacts.sh +++ b/distribution/build_and_upload_gcp_artifacts.sh @@ -39,13 +39,14 @@ fi echo "Building jars" sbt clean -sbt cloud_gcp/assembly -sbt cloud_gcp_submitter/assembly sbt flink/assembly sbt service/assembly -CLOUD_GCP_JAR="$CHRONON_ROOT_DIR/cloud_gcp/target/scala-2.12/cloud_gcp-assembly-0.1.0-SNAPSHOT.jar" -CLOUD_GCP_SUBMITTER_JAR="$CHRONON_ROOT_DIR/cloud_gcp_submitter/target/scala-2.12/cloud_gcp_submitter-assembly-0.1.0-SNAPSHOT.jar" +bazel build //cloud_gcp:lib_deploy.jar +bazel build //cloud_gcp:submitter_deploy.jar + +CLOUD_GCP_JAR="$CHRONON_ROOT_DIR/bazel-bin/cloud_gcp/lib_deploy.jar" +CLOUD_GCP_SUBMITTER_JAR="$CHRONON_ROOT_DIR/bazel-bin/cloud_gcp/submitter_deploy.jar" FLINK_JAR="$CHRONON_ROOT_DIR/flink/target/scala-2.12/flink-assembly-0.1.0-SNAPSHOT.jar" SERVICE_JAR="$CHRONON_ROOT_DIR/service/target/scala-2.12/service-0.1.0-SNAPSHOT.jar" From 53dde611f5f7e47f1f1e96e62587f801bb8316e4 Mon Sep 17 00:00:00 2001 From: tchow-zlai Date: Sat, 1 Feb 2025 00:29:12 -0800 Subject: [PATCH 2/2] update name Co-authored-by: Thomas Chow --- api/py/ai/chronon/repo/run.py | 4 ++-- cloud_gcp/BUILD.bazel | 14 +++++++------- online/BUILD.bazel | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/api/py/ai/chronon/repo/run.py b/api/py/ai/chronon/repo/run.py index ecbe7d4a53..c35795befa 100755 --- a/api/py/ai/chronon/repo/run.py +++ b/api/py/ai/chronon/repo/run.py @@ -130,10 +130,10 @@ # GCP DATAPROC SPECIFIC CONSTANTS DATAPROC_ENTRY = "ai.chronon.integrations.cloud_gcp.DataprocSubmitter" -ZIPLINE_ONLINE_JAR_DEFAULT = "lib_deploy.jar" +ZIPLINE_ONLINE_JAR_DEFAULT = "cloud_gcp_lib_deploy.jar" ZIPLINE_ONLINE_CLASS_DEFAULT = "ai.chronon.integrations.cloud_gcp.GcpApiImpl" ZIPLINE_FLINK_JAR_DEFAULT = "flink-assembly-0.1.0-SNAPSHOT.jar" -ZIPLINE_DATAPROC_SUBMITTER_JAR = "submitter_deploy.jar" +ZIPLINE_DATAPROC_SUBMITTER_JAR = "cloud_gcp_submitter_deploy.jar" class DataprocJobType(Enum): diff --git a/cloud_gcp/BUILD.bazel b/cloud_gcp/BUILD.bazel index 09b3b44946..5f35cf37e0 100644 --- a/cloud_gcp/BUILD.bazel +++ b/cloud_gcp/BUILD.bazel @@ -1,5 +1,5 @@ scala_library( - name = "lib", + name = "cloud_gcp_lib", srcs = glob(["src/main/**/*.scala"]), visibility = ["//visibility:public"], deps = [ @@ -37,13 +37,13 @@ scala_library( ) jvm_binary( - name = "submitter", + name = "cloud_gcp_submitter", main_class = "ai.chronon.integrations.cloud_gcp.DataprocSubmitter", - runtime_deps = [":lib"], + runtime_deps = [":cloud_gcp_lib"], ) test_deps = [ - ":lib", + ":cloud_gcp_lib", "//api:thrift", "//api:lib", "//online:lib", @@ -82,7 +82,7 @@ test_deps = [ ] scala_library( - name = "test-lib", + name = "cloud_gcp_test_lib", srcs = glob(["src/test/**/*.scala"]), visibility = ["//visibility:public"], deps = test_deps, @@ -90,10 +90,10 @@ scala_library( scala_test_suite( - name = "test", + name = "cloud_gcp_test", srcs = glob(["src/test/**/*.scala"]), # defined in prelude_bazel file jvm_flags = _JVM_FLAGS_FOR_ACCESSING_BASE_JAVA_CLASSES, visibility = ["//visibility:public"], - deps = test_deps + [":test-lib"], + deps = test_deps + [":cloud_gcp_test_lib"], ) diff --git a/online/BUILD.bazel b/online/BUILD.bazel index f0a470cf26..b0868cfc7c 100644 --- a/online/BUILD.bazel +++ b/online/BUILD.bazel @@ -18,6 +18,7 @@ scala_library( maven_artifact("com.github.ben-manes.caffeine:caffeine"), maven_artifact("com.fasterxml.jackson.core:jackson-core"), maven_artifact("com.fasterxml.jackson.core:jackson-databind"), + scala_artifact("com.fasterxml.jackson.module:jackson-module-scala"), scala_artifact("com.softwaremill.sttp.client3:core"), scala_artifact("com.softwaremill.sttp.model:core"), scala_artifact("com.softwaremill.sttp.shared:core"),