feat(scio-cosmosdb): feat(azure-cosmosdb): Add support for cosmosdb w…

…ith Core (SQL) API Refs: spotify#4675
Miuler · Jan 27, 2023 · f582184 · f582184
1 parent a68b370
commit f582184
Show file tree

Hide file tree

Showing 14 changed files with 525 additions and 119 deletions.
diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -1,4 +1,4 @@
-version = "3.7.0"
+version = "3.7.1"
 runner.dialect = scala213source3
 
 fileOverride {

diff --git a/README.md b/README.md
@@ -78,6 +78,7 @@ Scio includes the following artifacts:
 - `scio-avro`: add-on for Avro, can also be used standalone
 - `scio-google-cloud-platform`: add-on for Google Cloud IO's: BigQuery, Bigtable, Pub/Sub, Datastore, Spanner
 - `scio-cassandra*`: add-ons for Cassandra
+- `scio-cosmosdb`: add-ons for Azure CosmosDB Core (SQL) API
 - `scio-elasticsearch*`: add-ons for Elasticsearch
 - `scio-extra`: extra utilities for working with collections, Breeze, etc., best effort support
 - `scio-jdbc`: add-on for JDBC IO

diff --git a/build.sbt b/build.sbt
@@ -86,6 +86,10 @@ val algebraVersion = "2.9.0"
 val annoy4sVersion = "0.10.0"
 val annoyVersion = "0.2.6"
 val breezeVersion = "2.1.0"
+val bsonVersion = "4.8.1"
+val cosmosVersion = "4.37.1"
+val cosmosContainerVersion = "1.17.5"
+val scribeVersion = "3.10.7"
 val caffeineVersion = "2.9.3"
 val cassandraDriverVersion = "3.11.3"
 val cassandraVersion = "3.11.13"
@@ -156,9 +160,7 @@ ThisBuild / tpolecatDevModeOptions ~= { opts =>
   opts.filterNot(excludes).union(extras)
 }
 
-ThisBuild / doc / tpolecatDevModeOptions ++= Set(
-  Scalac.docNoJavaCommentOption
-)
+ThisBuild / doc / tpolecatDevModeOptions ++= Set(Scalac.docNoJavaCommentOption)
 
 ThisBuild / scalafixScalaBinaryVersion := CrossVersion.binaryScalaVersion(scalaVersion.value)
 val excludeLint = SettingKey[Set[Def.KeyedInitialize[_]]]("excludeLintKeys")
@@ -199,9 +201,7 @@ lazy val keepExistingHeader =
   HeaderCommentStyle.cStyleBlockComment.copy(commentCreator = new CommentCreator() {
     override def apply(text: String, existingText: Option[String]): String =
       existingText
-        .getOrElse(
-          HeaderCommentStyle.cStyleBlockComment.commentCreator(text)
-        )
+        .getOrElse(HeaderCommentStyle.cStyleBlockComment.commentCreator(text))
         .trim()
   })
 
@@ -217,9 +217,7 @@ val commonSettings = Def
     inTask(doc)(TpolecatPlugin.projectSettings),
     javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint:unchecked"),
     Compile / doc / javacOptions := Seq("-source", "1.8"),
-    excludeDependencies ++= Seq(
-      "org.apache.beam" % "beam-sdks-java-io-kafka"
-    ),
+    excludeDependencies ++= Seq("org.apache.beam" % "beam-sdks-java-io-kafka"),
     resolvers ++= Resolver.sonatypeOssRepos("public"),
     Test / javaOptions += "-Dscio.ignoreVersionWarning=true",
     Test / testOptions += Tests.Argument("-oD"),
@@ -240,9 +238,7 @@ val commonSettings = Def
     coverageHighlighting := true,
     licenses := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")),
     homepage := Some(url("https://github.com/spotify/scio")),
-    scmInfo := Some(
-      ScmInfo(url("https://github.com/spotify/scio"), "scm:git:[email protected]:spotify/scio.git")
-    ),
+    scmInfo := Some(ScmInfo(url("https://github.com/spotify/scio"), "scm:git:[email protected]:spotify/scio.git")),
     developers := List(
       Developer(
         id = "sinisa_lyh",
@@ -367,21 +363,15 @@ lazy val macroSettings = Def.settings(
   libraryDependencies ++= {
     VersionNumber(scalaVersion.value) match {
       case v if v.matchesSemVer(SemanticSelector("2.12.x")) =>
-        Seq(
-          compilerPlugin(
-            ("org.scalamacros" % "paradise" % scalaMacrosVersion).cross(CrossVersion.full)
-          )
-        )
+        Seq(compilerPlugin(("org.scalamacros" % "paradise" % scalaMacrosVersion).cross(CrossVersion.full)))
       case _ => Nil
     }
   },
   // see MacroSettings.scala
   scalacOptions += "-Xmacro-settings:cache-implicit-schemas=true"
 )
 
-lazy val directRunnerDependencies = Seq(
-  "org.apache.beam" % "beam-runners-direct-java" % beamVersion % Runtime
-)
+lazy val directRunnerDependencies = Seq("org.apache.beam" % "beam-runners-direct-java" % beamVersion % Runtime)
 lazy val dataflowRunnerDependencies = Seq(
   "org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % beamVersion % Runtime
 )
@@ -402,27 +392,28 @@ lazy val flinkRunnerDependencies = Seq(
 lazy val beamRunners = settingKey[String]("beam runners")
 lazy val beamRunnersEval = settingKey[Seq[ModuleID]]("beam runners")
 
-def beamRunnerSettings: Seq[Setting[_]] = Seq(
-  beamRunners := "",
-  beamRunnersEval := {
-    Option(beamRunners.value)
-      .filter(_.nonEmpty)
-      .orElse(sys.props.get("beamRunners"))
-      .orElse(sys.env.get("BEAM_RUNNERS"))
-      .map(_.split(","))
-      .map {
-        _.flatMap {
-          case "DirectRunner"   => directRunnerDependencies
-          case "DataflowRunner" => dataflowRunnerDependencies
-          case "SparkRunner"    => sparkRunnerDependencies
-          case "FlinkRunner"    => flinkRunnerDependencies
-          case _                => Nil
-        }.toSeq
-      }
-      .getOrElse(directRunnerDependencies)
-  },
-  libraryDependencies ++= beamRunnersEval.value
-)
+def beamRunnerSettings: Seq[Setting[_]] =
+  Seq(
+    beamRunners := "",
+    beamRunnersEval := {
+      Option(beamRunners.value)
+        .filter(_.nonEmpty)
+        .orElse(sys.props.get("beamRunners"))
+        .orElse(sys.env.get("BEAM_RUNNERS"))
+        .map(_.split(","))
+        .map {
+          _.flatMap {
+            case "DirectRunner"   => directRunnerDependencies
+            case "DataflowRunner" => dataflowRunnerDependencies
+            case "SparkRunner"    => sparkRunnerDependencies
+            case "FlinkRunner"    => flinkRunnerDependencies
+            case _                => Nil
+          }.toSeq
+        }
+        .getOrElse(directRunnerDependencies)
+    },
+    libraryDependencies ++= beamRunnersEval.value
+  )
 
 ThisBuild / PB.protocVersion := protobufVersion
 lazy val scopedProtobufSettings = Def.settings(
@@ -454,15 +445,12 @@ def splitTests(tests: Seq[TestDefinition], filter: Seq[String], forkOptions: For
 
 lazy val root: Project = Project("scio", file("."))
   .settings(commonSettings)
-  .settings(
-    publish / skip := true,
-    mimaPreviousArtifacts := Set.empty,
-    assembly / aggregate := false
-  )
+  .settings(publish / skip := true, mimaPreviousArtifacts := Set.empty, assembly / aggregate := false)
   .aggregate(
     `scio-avro`,
     `scio-cassandra3`,
     `scio-core`,
+    `scio-cosmosdb`,
     `scio-elasticsearch6`,
     `scio-elasticsearch7`,
     `scio-elasticsearch8`,
@@ -548,6 +536,25 @@ lazy val `scio-core`: Project = project
     buildInfoPackage := "com.spotify.scio"
   )
 
+lazy val `scio-cosmosdb`: Project = project
+  .in(file("scio-cosmosdb"))
+  .configs(IntegrationTest)
+  .settings(itSettings)
+  .settings(commonSettings)
+  .settings(publishSettings)
+  .dependsOn(`scio-core`, `scio-test` % "test;it")
+  .settings(
+    //scalacOptions ++= Seq("-deprecation", "-feature", "-unchecked", "-Xsource:3"), // , "-Ymacro-annotations"
+    scalacOptions ++= Seq("-Xsource:3"),
+    libraryDependencies ++= Seq(
+      "org.mongodb" % "bson" % bsonVersion,
+      "com.azure" % "azure-cosmos" % cosmosVersion,
+      "org.testcontainers" % "azure" % cosmosContainerVersion % IntegrationTest,
+      "com.outr" %% "scribe" % scribeVersion % IntegrationTest,
+      "com.outr" %% "scribe-slf4j" % scribeVersion % IntegrationTest
+    )
+  )
+
 lazy val `scio-test`: Project = project
   .in(file("scio-test"))
   .settings(commonSettings)
@@ -597,10 +604,7 @@ lazy val `scio-test`: Project = project
     )
   )
   .configs(IntegrationTest)
-  .dependsOn(
-    `scio-core` % "test->test;compile->compile;it->it",
-    `scio-avro` % "compile->test;it->it"
-  )
+  .dependsOn(`scio-core` % "test->test;compile->compile;it->it", `scio-avro` % "compile->test;it->it")
 
 lazy val `scio-macros`: Project = project
   .in(file("scio-macros"))
@@ -621,9 +625,7 @@ lazy val `scio-macros`: Project = project
 
 lazy val `scio-avro`: Project = project
   .in(file("scio-avro"))
-  .dependsOn(
-    `scio-core` % "compile;it->it"
-  )
+  .dependsOn(`scio-core` % "compile;it->it")
   .configs(IntegrationTest)
   .settings(commonSettings)
   .settings(publishSettings)
@@ -658,11 +660,7 @@ lazy val `scio-avro`: Project = project
 
 lazy val `scio-google-cloud-platform`: Project = project
   .in(file("scio-google-cloud-platform"))
-  .dependsOn(
-    `scio-core` % "compile;it->it",
-    `scio-avro` % "test",
-    `scio-test` % "test;it"
-  )
+  .dependsOn(`scio-core` % "compile;it->it", `scio-avro` % "test", `scio-test` % "test;it")
   .configs(IntegrationTest)
   .settings(commonSettings)
   .settings(publishSettings)
@@ -721,10 +719,7 @@ lazy val `scio-google-cloud-platform`: Project = project
 
 lazy val `scio-cassandra3`: Project = project
   .in(file("scio-cassandra/cassandra3"))
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test;it"
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test;it")
   .configs(IntegrationTest)
   .settings(commonSettings)
   .settings(publishSettings)
@@ -755,10 +750,7 @@ lazy val `scio-cassandra3`: Project = project
 
 lazy val `scio-elasticsearch6`: Project = project
   .in(file("scio-elasticsearch/es6"))
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test"
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test")
   .settings(commonSettings)
   .settings(publishSettings)
   .settings(
@@ -800,17 +792,11 @@ lazy val `scio-elasticsearch7`: Project = project
       "org.scalatest" %% "scalatest" % scalatestVersion % Test
     )
   )
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test"
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test")
 
 lazy val `scio-elasticsearch8`: Project = project
   .in(file("scio-elasticsearch/es8"))
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test,it"
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test,it")
   .configs(IntegrationTest)
   .settings(commonSettings)
   .settings(publishSettings)
@@ -893,10 +879,7 @@ lazy val `scio-extra`: Project = project
 
 lazy val `scio-grpc`: Project = project
   .in(file("scio-grpc"))
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test"
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test")
   .settings(commonSettings)
   .settings(publishSettings)
   .settings(protobufSettings)
@@ -913,10 +896,7 @@ lazy val `scio-grpc`: Project = project
 
 lazy val `scio-jdbc`: Project = project
   .in(file("scio-jdbc"))
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test"
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test")
   .settings(commonSettings)
   .settings(publishSettings)
   .settings(
@@ -930,10 +910,7 @@ lazy val `scio-jdbc`: Project = project
 
 lazy val `scio-neo4j`: Project = project
   .in(file("scio-neo4j"))
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test,it"
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test,it")
   .configs(IntegrationTest)
   .settings(commonSettings)
   .settings(itSettings)
@@ -956,11 +933,7 @@ val ensureSourceManaged = taskKey[Unit]("ensureSourceManaged")
 
 lazy val `scio-parquet`: Project = project
   .in(file("scio-parquet"))
-  .dependsOn(
-    `scio-core`,
-    `scio-avro`,
-    `scio-test` % "test->test"
-  )
+  .dependsOn(`scio-core`, `scio-avro`, `scio-test` % "test->test")
   .settings(commonSettings)
   .settings(publishSettings)
   .settings(
@@ -1005,11 +978,7 @@ lazy val `scio-parquet`: Project = project
 
 lazy val `scio-tensorflow`: Project = project
   .in(file("scio-tensorflow"))
-  .dependsOn(
-    `scio-avro`,
-    `scio-core`,
-    `scio-test` % "test->test"
-  )
+  .dependsOn(`scio-avro`, `scio-core`, `scio-test` % "test->test")
   .settings(commonSettings)
   .settings(publishSettings)
   .settings(itSettings)
@@ -1064,10 +1033,7 @@ lazy val `scio-examples`: Project = project
   .settings(
     publish / skip := true,
     mimaPreviousArtifacts := Set.empty,
-    tpolecatExcludeOptions ++= Set(
-      ScalacOptions.warnUnusedLocals,
-      ScalacOptions.privateWarnUnusedLocals
-    ),
+    tpolecatExcludeOptions ++= Set(ScalacOptions.warnUnusedLocals, ScalacOptions.privateWarnUnusedLocals),
     libraryDependencies ++= Seq(
       // compile
       "com.chuusai" %% "shapeless" % shapelessVersion,
@@ -1131,11 +1097,7 @@ lazy val `scio-examples`: Project = project
 
 lazy val `scio-repl`: Project = project
   .in(file("scio-repl"))
-  .dependsOn(
-    `scio-core`,
-    `scio-google-cloud-platform`,
-    `scio-extra`
-  )
+  .dependsOn(`scio-core`, `scio-google-cloud-platform`, `scio-extra`)
   .settings(commonSettings)
   .settings(publishSettings)
   .settings(assemblySettings)
@@ -1185,10 +1147,7 @@ lazy val `scio-repl`: Project = project
 lazy val `scio-jmh`: Project = project
   .in(file("scio-jmh"))
   .enablePlugins(JmhPlugin)
-  .dependsOn(
-    `scio-core`,
-    `scio-avro`
-  )
+  .dependsOn(`scio-core`, `scio-avro`)
   .settings(commonSettings)
   .settings(macroSettings)
   .settings(
@@ -1209,11 +1168,7 @@ lazy val `scio-jmh`: Project = project
 
 lazy val `scio-smb`: Project = project
   .in(file("scio-smb"))
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test;it",
-    `scio-avro` % IntegrationTest
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test;it", `scio-avro` % IntegrationTest)
   .configs(IntegrationTest)
   .settings(commonSettings)
   .settings(publishSettings)
@@ -1273,10 +1228,7 @@ lazy val `scio-smb`: Project = project
 
 lazy val `scio-redis`: Project = project
   .in(file("scio-redis"))
-  .dependsOn(
-    `scio-core`,
-    `scio-test` % "test"
-  )
+  .dependsOn(`scio-core`, `scio-test` % "test")
   .settings(commonSettings)
   .settings(publishSettings)
   .settings(itSettings)
@@ -1359,6 +1311,7 @@ lazy val siteSettings = Def.settings(
     `scio-avro`,
     `scio-google-cloud-platform`,
     `scio-cassandra3`,
+    `scio-cosmosdb`,
     `scio-elasticsearch8`,
     `scio-extra`,
     `scio-jdbc`,