Merge pull request #675 from typelevel/topic/backend-oracle-mssql

Modularised and added Oracle and SQL Server backends
typelevel · Nov 30, 2024 · d6bac44 · d6bac44
2 parents f3aaca8 + 6dc639c
commit d6bac44
Show file tree

Hide file tree

Showing 176 changed files with 13,771 additions and 654 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -61,6 +61,9 @@ jobs:
       - name: Check Headers
         run: sbt 'project ${{ matrix.project }}' '++ ${{ matrix.scala }}' headerCheckAll
 
+      - name: Start up test databases
+        run: docker compose up --force-recreate -d --wait --quiet-pull
+
       - name: Check headers
         if: matrix.java == 'temurin@11' && matrix.os == 'ubuntu-latest'
         run: sbt 'project ${{ matrix.project }}' '++ ${{ matrix.scala }}' headerCheckAll
@@ -86,11 +89,11 @@ jobs:
 
       - name: Make target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-        run: mkdir -p modules/skunk/js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/sql/js/target modules/skunk/jvm/target modules/core/.js/target modules/circe/.js/target modules/sql/native/target modules/skunk/native/target modules/generic/.js/target modules/core/.jvm/target modules/sql/jvm/target modules/circe/.native/target modules/generic/.native/target project/target
+        run: mkdir -p modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/doobie-mssql/target modules/sql-pg/js/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target
 
       - name: Compress target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-        run: tar cf targets.tar modules/skunk/js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/sql/js/target modules/skunk/jvm/target modules/core/.js/target modules/circe/.js/target modules/sql/native/target modules/skunk/native/target modules/generic/.js/target modules/core/.jvm/target modules/sql/jvm/target modules/circe/.native/target modules/generic/.native/target project/target
+        run: tar cf targets.tar modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/doobie-mssql/target modules/sql-pg/js/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target
 
       - name: Upload target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
@@ -279,6 +282,9 @@ jobs:
         if: matrix.java == 'temurin@11' && steps.setup-java-temurin-11.outputs.cache-hit == 'false'
         run: sbt +update
 
+      - name: Start up test databases
+        run: docker compose up --force-recreate -d --wait --quiet-pull
+
       - run: sbt coverage rootJVM/test coverageReport
 
       - uses: codecov/codecov-action@v3

diff --git a/README.md b/README.md
@@ -23,14 +23,15 @@ backing data, and cursors into that data. It supports in-memory, DB-backed, and
 Grackle is structured as a compiler/interpreter. Queries are type-checked against a GraphQL schema and compiled into
 an internal query algebra. The query algebra may be further compiled in a backend-specific way to materialize data. In
 particular it can be compiled to efficient SQL and in that regard currently supports Postgres via
-[Doobie](https://tpolecat.github.io/doobie/) or [Skunk](https://typelevel.org/skunk/).
+[Doobie](https://tpolecat.github.io/doobie/) or [Skunk](https://typelevel.org/skunk/) and Oracle and SQL Server via
+Doobie.
 
 Grackle is an [Apache 2.0 licensed](https://www.apache.org/licenses/LICENSE-2.0) Typelevel project and is available
 for Scala 2/3 and for [Scala.js](https://www.scala-js.org/) and [Scala Native](https://scala-native.org/en/stable/).
 
 Work has been generously sponsored by
-[Aura/Gemini](https://www.aura-astronomy.org/centers/nsfs-oir-lab/gemini-observatory/) and [ITV](https://www.itv.com)
-over the last four years.
+[Aura/Gemini](https://www.aura-astronomy.org/centers/nsfs-oir-lab/gemini-observatory/), [ITV](https://www.itv.com)
+and [imbus AG](https://www.imbus.de/) over the last five years.
 
 ## Getting Started
 
@@ -55,8 +56,24 @@ libraryDependencies += "org.typelevel" %% "grackle-doobie-pg" % "0.22.0"
 
 // Optional: support for Postgres backend via Skunk
 libraryDependencies += "org.typelevel" %% "grackle-skunk" % "0.22.0"
+
+// Optional: support for Oracle backend via Doobie (JVM only)
+libraryDependencies += "org.typelevel" %% "grackle-doobie-oracle" % "0.22.0"
+
+// Optional: support for SQL Server backend via Doobie (JVM only)
+libraryDependencies += "org.typelevel" %% "grackle-doobie-mssql" % "0.22.0"
 ```
 
+## Running tests for database backed mappings
+
+Database backed mappings are tested against dockerized instances of Postgres, Oracle and SQL Server. This requires
+Docker 2.20.2 or later to be installed and running on the test machine.
+
+Running tests (eg. `rootJVM/test`) will automatically spin up the relevant containers, which will stay up and can be
+reused across multiple tests runs, significantly speeding up the test cycle. I recommend running `allUp` initially, to
+pull images and initialise databases, before running tests the first time. Note that Oracle in particular takes quite
+a long time to initialise, so expect `allUp` to take several minutes to complete the first time around.
+
 ## Community
 
 Grackle is proud to be a [Typelevel](https://typelevel.org/) project. We are committed to providing a friendly, safe

diff --git a/build.sbt b/build.sbt
@@ -1,5 +1,6 @@
 import nl.zolotko.sbt.jfr.{JfrRecording, JfrRecorderOptions}
 import scala.concurrent.duration.DurationInt
+import scala.sys.process._
 
 val catsVersion            = "2.11.0"
 val catsParseVersion       = "1.0.0"
@@ -9,28 +10,29 @@ val disciplineMunitVersion = "2.0.0-M3"
 val doobieVersion          = "1.0.0-RC6"
 val fs2Version             = "3.11.0"
 val http4sVersion          = "0.23.29"
-val jnrUnixsocketVersion   = "0.38.23"
 val kindProjectorVersion   = "0.13.3"
 val literallyVersion       = "1.1.0"
 val logbackVersion         = "1.5.12"
 val log4catsVersion        = "2.7.0"
+val mssqlDriverVersion     = "12.8.1.jre11"
 val munitVersion           = "1.0.0-M11"
 val munitCatsEffectVersion = "2.0.0"
 val munitScalaCheckVersion = "1.0.0-M11"
+val oracleDriverVersion    = "23.5.0.24.07"
 val skunkVersion           = "0.6.4"
 val shapeless2Version      = "2.3.11"
 val shapeless3Version      = "3.4.1"
 val sourcePosVersion       = "1.1.0"
 val typenameVersion        = "1.1.0"
-val whaleTailVersion       = "0.0.12"
 
 val Scala2 = "2.13.15"
 val Scala3 = "3.3.4"
+
 ThisBuild / scalaVersion        := Scala2
 ThisBuild / crossScalaVersions  := Seq(Scala2, Scala3)
 ThisBuild / tlJdkRelease        := Some(11)
 
-ThisBuild / tlBaseVersion    := "0.22"
+ThisBuild / tlBaseVersion    := "0.23"
 ThisBuild / startYear        := Some(2019)
 ThisBuild / licenses         := Seq(License.Apache2)
 ThisBuild / developers       := List(
@@ -42,10 +44,16 @@ ThisBuild / tlFatalWarnings         := true
 ThisBuild / tlCiScalafmtCheck       := false
 ThisBuild / tlCiReleaseBranches     := Seq("main")
 ThisBuild / githubWorkflowBuild     ~= { steps =>
-  WorkflowStep.Sbt(
-    commands = List("headerCheckAll"),
-    name = Some("Check Headers"),
-  ) +: steps
+  Seq(
+    WorkflowStep.Sbt(
+      commands = List("headerCheckAll"),
+      name = Some("Check Headers")
+    ),
+    WorkflowStep.Run(
+      commands = List("docker compose up --force-recreate -d --wait --quiet-pull"),
+      name = Some("Start up test databases")
+    )
+  ) ++ steps
 }
 ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("11"))
 ThisBuild / tlBspCrossProjectPlatforms := Set(JVMPlatform)
@@ -58,14 +66,39 @@ ThisBuild / githubWorkflowAddedJobs +=
     sbtStepPreamble = Nil,
     steps = githubWorkflowJobSetup.value.toList ++
       List(
+        WorkflowStep.Run(
+          commands = List("docker compose up --force-recreate -d --wait --quiet-pull"),
+          name = Some("Start up test databases")
+        ),
         WorkflowStep.Sbt(List("coverage", "rootJVM/test", "coverageReport")),
         WorkflowStep.Use(UseRef.Public("codecov", "codecov-action", "v3"))
       )
   )
 
-
 ThisBuild / tlSitePublishBranch := Some("main")
 
+lazy val allUp = taskKey[Unit]("Start all docker compose services")
+lazy val allStop = taskKey[Unit]("Stop all docker compose services")
+lazy val pgUp = taskKey[Unit]("Start Postgres")
+lazy val pgStop = taskKey[Unit]("Stop Postgres")
+lazy val oracleUp = taskKey[Unit]("Start Oracle")
+lazy val oracleStop = taskKey[Unit]("Stop Oracle")
+lazy val mssqlUp = taskKey[Unit]("Start SQL Server")
+lazy val mssqlStop = taskKey[Unit]("Stop SQL Server")
+
+ThisBuild / allUp := runDocker("docker compose up -d --wait --quiet-pull")
+ThisBuild / allStop := runDocker("docker compose stop")
+ThisBuild / pgUp := runDocker("docker compose up -d --wait --quiet-pull postgres")
+ThisBuild / pgStop := runDocker("docker compose stop postgres")
+ThisBuild / oracleUp := runDocker("docker compose up -d --wait --quiet-pull oracle")
+ThisBuild / oracleStop := runDocker("docker compose stop oracle")
+ThisBuild / mssqlUp := runDocker("docker compose up -d --wait --quiet-pull mssql")
+ThisBuild / mssqlStop := runDocker("docker compose stop mssql")
+
+def runDocker(cmd: String): Unit = {
+  require(cmd.! == 0, s"docker indicated an error")
+}
+
 lazy val commonSettings = Seq(
   //scalacOptions --= Seq("-Wunused:params", "-Wunused:imports", "-Wunused:patvars", "-Wdead-code", "-Wunused:locals", "-Wunused:privates", "-Wunused:implicits"),
   scalacOptions ++= Seq("-Xlint:-pattern-shadow").filterNot(_ => tlIsScala3.value),
@@ -77,6 +110,7 @@ lazy val commonSettings = Seq(
     "org.typelevel" %%% "munit-cats-effect" % munitCatsEffectVersion % "test",
     "io.circe"      %%% "circe-literal"     % circeVersion % "test",
     "io.circe"      %%% "circe-jawn"        % circeVersion % "test",
+    "io.circe"      %%% "circe-parser"      % circeVersion % "test",
   ) ++ Seq(
     compilerPlugin("org.typelevel" %% "kind-projector" % kindProjectorVersion cross CrossVersion.full),
   ).filterNot(_ => tlIsScala3.value),
@@ -103,8 +137,12 @@ lazy val commonSettings = Seq(
 lazy val modules: List[CompositeProject] = List(
   core,
   circe,
-  sql,
-  doobie,
+  sqlcore,
+  sqlpg,
+  doobiecore,
+  doobiepg,
+  doobieoracle,
+  doobiemssql,
   skunk,
   generic,
   docs,
@@ -162,46 +200,94 @@ lazy val buildInfo = crossProject(JVMPlatform, JSPlatform, NativePlatform)
     buildInfoKeys += "baseDirectory" -> (LocalRootProject / baseDirectory).value.toString
   )
 
-lazy val sql = crossProject(JVMPlatform, JSPlatform, NativePlatform)
-  .crossType(CrossType.Full)
-  .in(file("modules/sql"))
+lazy val sqlcore = crossProject(JVMPlatform, JSPlatform, NativePlatform)
+  .crossType(CrossType.Pure)
+  .in(file("modules/sql-core"))
   .enablePlugins(AutomateHeaderPlugin)
   .disablePlugins(RevolverPlugin)
   .dependsOn(core % "test->test;compile->compile", circe, buildInfo % Test)
   .settings(commonSettings)
   .settings(
-    name := "grackle-sql",
+    name := "grackle-sql-core",
     libraryDependencies ++= Seq(
       "io.circe"          %%% "circe-generic"      % circeVersion % "test",
       "co.fs2"            %%% "fs2-io"             % fs2Version % "test",
     )
   )
-  .platformsSettings(JVMPlatform, JSPlatform)(
-    libraryDependencies ++= Seq(
-      "io.chrisdavenport" %%% "whale-tail-manager" % whaleTailVersion % "test",
-    )
+
+lazy val sqlpg = crossProject(JVMPlatform, JSPlatform, NativePlatform)
+  .crossType(CrossType.Full)
+  .in(file("modules/sql-pg"))
+  .enablePlugins(AutomateHeaderPlugin)
+  .disablePlugins(RevolverPlugin)
+  .dependsOn(sqlcore % "test->test;compile->compile", circe)
+  .settings(commonSettings)
+  .settings(
+    name := "grackle-sql-pg",
   )
-  .jvmSettings(
+
+lazy val doobiecore = project
+  .in(file("modules/doobie-core"))
+  .enablePlugins(AutomateHeaderPlugin)
+  .disablePlugins(RevolverPlugin)
+  .dependsOn(sqlcore.jvm % "test->test;compile->compile", circe.jvm)
+  .settings(commonSettings)
+  .settings(
+    name := "grackle-doobie-core",
+    Test / fork := true,
+    Test / parallelExecution := false,
     libraryDependencies ++= Seq(
-      "com.github.jnr"    % "jnr-unixsocket"      % jnrUnixsocketVersion % "test"
+      "org.tpolecat"   %% "doobie-core"     % doobieVersion,
+      "org.typelevel"  %% "log4cats-core"   % log4catsVersion,
+      "ch.qos.logback" %  "logback-classic" % logbackVersion % "test"
     )
   )
 
-lazy val doobie = project
+lazy val doobiepg = project
   .in(file("modules/doobie-pg"))
   .enablePlugins(AutomateHeaderPlugin)
   .disablePlugins(RevolverPlugin)
-  .dependsOn(sql.jvm % "test->test;compile->compile", circe.jvm)
+  .dependsOn(doobiecore % "test->test;compile->compile", sqlpg.jvm % "test->test;compile->compile")
   .settings(commonSettings)
   .settings(
     name := "grackle-doobie-pg",
     Test / fork := true,
     Test / parallelExecution := false,
+    Test / testOptions += Tests.Setup(_ => runDocker("docker compose up -d --wait --quiet-pull postgres")),
     libraryDependencies ++= Seq(
-      "org.tpolecat"   %% "doobie-core"           % doobieVersion,
-      "org.tpolecat"   %% "doobie-postgres-circe" % doobieVersion,
-      "org.typelevel"  %% "log4cats-core"         % log4catsVersion,
-      "ch.qos.logback" %  "logback-classic"       % logbackVersion % "test"
+      "org.tpolecat" %% "doobie-postgres-circe" % doobieVersion
+    )
+  )
+
+lazy val doobieoracle = project
+  .in(file("modules/doobie-oracle"))
+  .enablePlugins(AutomateHeaderPlugin)
+  .disablePlugins(RevolverPlugin)
+  .dependsOn(doobiecore % "test->test;compile->compile")
+  .settings(commonSettings)
+  .settings(
+    name := "grackle-doobie-oracle",
+    Test / fork := true,
+    Test / parallelExecution := false,
+    Test / testOptions += Tests.Setup(_ => runDocker("docker compose up -d --wait --quiet-pull oracle")),
+    libraryDependencies ++= Seq(
+      "com.oracle.database.jdbc" % "ojdbc8" % oracleDriverVersion
+    )
+  )
+
+lazy val doobiemssql = project
+  .in(file("modules/doobie-mssql"))
+  .enablePlugins(AutomateHeaderPlugin)
+  .disablePlugins(RevolverPlugin)
+  .dependsOn(doobiecore % "test->test;compile->compile")
+  .settings(commonSettings)
+  .settings(
+    name := "grackle-doobie-mssql",
+    Test / fork := true,
+    Test / parallelExecution := false,
+    Test / testOptions += Tests.Setup(_ => runDocker("docker compose up -d --wait --quiet-pull mssql")),
+    libraryDependencies ++= Seq(
+      "com.microsoft.sqlserver" % "mssql-jdbc" % mssqlDriverVersion
     )
   )
 
@@ -210,7 +296,7 @@ lazy val skunk = crossProject(JVMPlatform, JSPlatform, NativePlatform)
   .in(file("modules/skunk"))
   .enablePlugins(AutomateHeaderPlugin)
   .disablePlugins(RevolverPlugin)
-  .dependsOn(sql % "test->test;compile->compile", circe)
+  .dependsOn(sqlpg % "test->test;compile->compile", circe)
   .settings(commonSettings)
   .settings(
     name := "grackle-skunk",
@@ -223,6 +309,7 @@ lazy val skunk = crossProject(JVMPlatform, JSPlatform, NativePlatform)
   )
   .jvmSettings(
     Test / fork := true,
+    Test / testOptions += Tests.Setup(_ => runDocker("docker compose up -d --wait --quiet-pull postgres")),
     libraryDependencies ++= Seq(
       "ch.qos.logback" % "logback-classic" % logbackVersion % "test"
     )
@@ -247,10 +334,12 @@ lazy val generic = crossProject(JVMPlatform, JSPlatform, NativePlatform)
       })
   )
 
+import spray.revolver.Actions._
+
 lazy val demo = project
   .in(file("demo"))
   .enablePlugins(NoPublishPlugin, AutomateHeaderPlugin)
-  .dependsOn(buildInfo.jvm, core.jvm, generic.jvm, doobie)
+  .dependsOn(buildInfo.jvm, core.jvm, generic.jvm, doobiepg)
   .settings(commonSettings)
   .settings(
     name := "grackle-demo",
@@ -264,26 +353,29 @@ lazy val demo = project
       "org.http4s"        %% "http4s-ember-server" % http4sVersion,
       "org.http4s"        %% "http4s-ember-client" % http4sVersion,
       "org.http4s"        %% "http4s-circe"        % http4sVersion,
-      "org.http4s"        %% "http4s-dsl"          % http4sVersion,
-      "io.chrisdavenport" %% "whale-tail-manager"  % whaleTailVersion,
-      "com.github.jnr"    % "jnr-unixsocket"       % jnrUnixsocketVersion
-    )
+      "org.http4s"        %% "http4s-dsl"          % http4sVersion
+    ),
+    reStart := // Redefine reStart to depend on pgUp
+      Def.inputTask(reStart.evaluated)
+        .dependsOn(Compile / products)
+        .dependsOn(ThisBuild / pgUp)
+        .evaluated
   )
 
 lazy val benchmarks = project
   .in(file("benchmarks"))
   .dependsOn(core.jvm)
   .enablePlugins(NoPublishPlugin, AutomateHeaderPlugin, JmhPlugin)
   .settings(commonSettings)
-  .settings(    
+  .settings(
     coverageEnabled := false,
-)
+  )
 
 lazy val profile = project
   .in(file("profile"))
   .enablePlugins(NoPublishPlugin, AutomateHeaderPlugin)
   .dependsOn(core.jvm)
-  .dependsOn(doobie)
+  .dependsOn(doobiepg)
   .settings(commonSettings)
   .settings(
     jfrRecordings := Seq(
@@ -338,8 +430,12 @@ lazy val unidocs = project
     ScalaUnidoc / unidoc / unidocProjectFilter := inProjects(
       core.jvm,
       circe.jvm,
-      sql.jvm,
-      doobie,
+      sqlcore.jvm,
+      sqlpg.jvm,
+      doobiecore,
+      doobiepg,
+      doobieoracle,
+      doobiemssql,
       skunk.jvm,
       generic.jvm,
     )