diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts b/build-logic/src/main/kotlin/polaris-java.gradle.kts index e3b1767078..68f856a061 100644 --- a/build-logic/src/main/kotlin/polaris-java.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts @@ -18,6 +18,7 @@ */ import com.diffplug.spotless.FormatterFunc +import gradle.kotlin.dsl.accessors._610c9322e0f6ac91715e46c62199c37f.spotless import java.io.Serializable import net.ltgt.gradle.errorprone.errorprone import org.gradle.api.tasks.compile.JavaCompile @@ -39,20 +40,25 @@ apply() tasks.withType(JavaCompile::class.java).configureEach { options.compilerArgs.addAll(listOf("-Xlint:unchecked", "-Xlint:deprecation")) - options.errorprone.disableAllWarnings = true - options.errorprone.disableWarningsInGeneratedCode = true - options.errorprone.excludedPaths = - ".*/${project.layout.buildDirectory.get().asFile.relativeTo(projectDir)}/generated/.*" - options.errorprone.error( - "DefaultCharset", - "FallThrough", - "MissingCasesInEnumSwitch", - "MissingOverride", - "ModifiedButNotUsed", - "OrphanedFormatString", - "PatternMatchingInstanceof", - "StringCaseLocaleUsage", - ) + + if (project.extra.has("reused-project-dir")) { + options.errorprone.disableAllChecks = true + } else { + options.errorprone.disableAllWarnings = true + options.errorprone.disableWarningsInGeneratedCode = true + options.errorprone.excludedPaths = + ".*/${project.layout.buildDirectory.get().asFile.relativeTo(projectDir)}/generated/.*" + options.errorprone.error( + "DefaultCharset", + "FallThrough", + "MissingCasesInEnumSwitch", + "MissingOverride", + "ModifiedButNotUsed", + "OrphanedFormatString", + "PatternMatchingInstanceof", + "StringCaseLocaleUsage", + ) + } } tasks.register("compileAll").configure { @@ -150,42 +156,53 @@ tasks.withType(Jar::class).configureEach { } } -spotless { - java { - target("src/*/java/**/*.java") - googleJavaFormat() - licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt")) - endWithNewline() - custom( - "disallowWildcardImports", - object : Serializable, FormatterFunc { - override fun apply(text: String): String { - val regex = "~/import .*\\.\\*;/".toRegex() - if (regex.matches(text)) { - throw GradleException("Wildcard imports disallowed - ${regex.findAll(text)}") +if (!project.extra.has("reused-project-dir") && !gradle.ideSyncActive()) { + spotless { + java { + target("src/*/java/**/*.java") + googleJavaFormat() + licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt")) + endWithNewline() + custom( + "disallowWildcardImports", + object : Serializable, FormatterFunc { + override fun apply(text: String): String { + val regex = "~/import .*\\.\\*;/".toRegex() + if (regex.matches(text)) { + throw GradleException("Wildcard imports disallowed - ${regex.findAll(text)}") + } + return text } - return text - } - }, - ) - toggleOffOn() - } - kotlinGradle { - ktfmt().googleStyle() - licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$") - target("*.gradle.kts") - } - format("xml") { - target("src/**/*.xml", "src/**/*.xsd") - targetExclude("codestyle/copyright-header.xml") - eclipseWtp(com.diffplug.spotless.extra.wtp.EclipseWtpFormatterStep.XML) - .configFile(rootProject.file("codestyle/org.eclipse.wst.xml.core.prefs")) - // getting the license-header delimiter right is a bit tricky. - // licenseHeaderFile(rootProject.file("codestyle/copyright-header.xml"), '<^[!?].*$') + }, + ) + toggleOffOn() + } + scala { + scalafmt(requiredDependencyVersion("scalafmt")) + .configFile(rootProject.file("codestyle/scalafmt.conf").toString()) + licenseHeaderFile( + rootProject.file("codestyle/copyright-header-java.txt"), + "^(package|import) .*$", + ) + target("src/**/scala/**/*.scala") + } + kotlinGradle { + ktfmt().googleStyle() + licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$") + target("*.gradle.kts") + } + format("xml") { + target("src/**/*.xml", "src/**/*.xsd") + targetExclude("codestyle/copyright-header.xml") + eclipseWtp(com.diffplug.spotless.extra.wtp.EclipseWtpFormatterStep.XML) + .configFile(rootProject.file("codestyle/org.eclipse.wst.xml.core.prefs")) + // getting the license-header delimiter right is a bit tricky. + // licenseHeaderFile(rootProject.file("codestyle/copyright-header.xml"), '<^[!?].*$') + } } } -dependencies { errorprone(versionCatalogs.named("libs").findLibrary("errorprone").get()) } +dependencies { errorprone(requiredDependency("errorprone")) } java { withJavadocJar() diff --git a/build-logic/src/main/kotlin/polaris-root.gradle.kts b/build-logic/src/main/kotlin/polaris-root.gradle.kts index 96faa07b82..58fabdd95a 100644 --- a/build-logic/src/main/kotlin/polaris-root.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-root.gradle.kts @@ -41,7 +41,7 @@ spotless { } } -if (System.getProperty("idea.sync.active").toBoolean()) { +if (gradle.ideSyncActive()) { idea { module { isDownloadJavadoc = false // was 'true', but didn't work diff --git a/build-logic/src/main/kotlin/polaris-scala.gradle.kts b/build-logic/src/main/kotlin/polaris-scala.gradle.kts new file mode 100644 index 0000000000..f69cd4b0e8 --- /dev/null +++ b/build-logic/src/main/kotlin/polaris-scala.gradle.kts @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.publish.PublishingExtension +import org.gradle.api.publish.maven.MavenPublication +import org.gradle.api.tasks.bundling.Jar +import org.gradle.api.tasks.scala.ScalaCompile +import org.gradle.api.tasks.scala.ScalaDoc +import org.gradle.kotlin.dsl.* +import org.gradle.language.scala.tasks.KeepAliveMode + +plugins { + id("polaris-server") + scala +} + +tasks.withType().configureEach { + options.release = 21 + scalaCompileOptions.additionalParameters.add("-release:21") + sourceCompatibility = "21" + targetCompatibility = "21" +} + +tasks.withType().configureEach { + scalaCompileOptions.keepAliveMode = KeepAliveMode.DAEMON + scalaCompileOptions.encoding = "UTF-8" +} + +val scaladoc = tasks.named("scaladoc") +val scaladocJar = tasks.register("scaladocJar") + +scaladocJar.configure { + dependsOn(scaladoc) + val baseJar = tasks.getByName("jar") + from(scaladoc.get().destinationDir) + destinationDirectory = baseJar.destinationDirectory + archiveClassifier = "scaladoc" +} + +tasks.named("assemble").configure { dependsOn(scaladocJar) } + +configure { + publications { + withType(MavenPublication::class.java) { + if (name == "maven") { + artifact(scaladocJar) + } + } + } +} diff --git a/build-logic/src/main/kotlin/polaris-spark.gradle.kts b/build-logic/src/main/kotlin/polaris-spark.gradle.kts new file mode 100644 index 0000000000..9e166c0b81 --- /dev/null +++ b/build-logic/src/main/kotlin/polaris-spark.gradle.kts @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("polaris-client") + scala +} + +tasks.withType().configureEach { + options.release = 11 + scalaCompileOptions.additionalParameters.add("-release:11") + sourceCompatibility = "11" + targetCompatibility = "11" +} + +tasks.withType().configureEach { + scalaCompileOptions.keepAliveMode = KeepAliveMode.DAEMON + scalaCompileOptions.encoding = "UTF-8" +} + +val scaladoc = tasks.named("scaladoc") +val scaladocJar = tasks.register("scaladocJar") + +scaladocJar.configure { + dependsOn(scaladoc) + val baseJar = tasks.getByName("jar") + from(scaladoc.get().destinationDir) + destinationDirectory = baseJar.destinationDirectory + archiveClassifier = "scaladoc" +} + +tasks.named("assemble").configure { dependsOn(scaladocJar) } + +configure { + publications { + withType(MavenPublication::class.java) { + if (name == "maven") { + artifact(scaladocJar) + } + } + } +} + +val versions = sparkScalaVersionsForProject() + +scala { scalaVersion.set(versions.scalaFullVersion) } diff --git a/build-logic/src/main/kotlin/spark-scala.kt b/build-logic/src/main/kotlin/spark-scala.kt new file mode 100644 index 0000000000..79b7a10cc9 --- /dev/null +++ b/build-logic/src/main/kotlin/spark-scala.kt @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.lang.IllegalArgumentException +import org.gradle.api.Project +import org.gradle.api.artifacts.Dependency +import org.gradle.api.artifacts.ExternalModuleDependency +import org.gradle.api.artifacts.ModuleDependency +import org.gradle.api.tasks.testing.Test +import org.gradle.kotlin.dsl.exclude + +fun Project.prepareSparkScalaProject(): SparkScalaVersions { + val versions = sparkScalaVersionsForProject() + + forceJavaVersionForTests(versions.runtimeJavaVersion) + addSparkJvmOptions() + return versions +} + +fun Project.prepareScalaProject(): SparkScalaVersions { + return sparkScalaVersionsForProject() +} + +/** Resolves the Spark and Scala major/minor versions from the Gradle project name. */ +fun Project.sparkScalaVersionsForProject(): SparkScalaVersions { + val matchScala = ".*_(\\d+[.]\\d+)".toRegex().matchEntire(project.name)!! + val matchSpark = ".*-(\\d+[.]\\d+)_\\d+[.]\\d+".toRegex().matchEntire(project.name) + + val sparkMajorMinorVersion = if (matchSpark != null) matchSpark.groups[1]!!.value else "" + val scalaMajorMinorVersion = matchScala.groups[1]!!.value + + project.layout.buildDirectory.set(layout.buildDirectory.dir(scalaMajorMinorVersion).get()) + + return SparkScalaVersions( + sparkMajorMinorVersion, + scalaMajorMinorVersion, + if (sparkMajorMinorVersion.isNotBlank()) + resolveFullSparkVersion(sparkMajorMinorVersion, scalaMajorMinorVersion) + else "", + resolveFullScalaVersion(scalaMajorMinorVersion), + if (sparkMajorMinorVersion.isNotBlank()) javaVersionForSpark(sparkMajorMinorVersion) else 0, + ) +} + +/** + * Apply the `sparkFullVersion` as a `strictly` version constraint and apply [withSparkExcludes] on + * the current [Dependency]. + */ +fun ModuleDependency.sparkDependencyAndExcludes(sparkScala: SparkScalaVersions): ModuleDependency { + val dep = this as ExternalModuleDependency + dep.version { strictly(sparkScala.sparkFullVersion) } + return this.withSparkExcludes() +} + +/** Apply a bunch of common dependency-exclusion to the current Spark [Dependency]. */ +fun ModuleDependency.withSparkExcludes(): ModuleDependency { + return this.exclude("commons-logging", "commons-logging") + .exclude("log4j", "log4j") + .exclude("org.slf4j", "slf4j-log4j12") + .exclude("org.slf4j", "slf4j-reload4j") + .exclude("org.eclipse.jetty", "jetty-util") + .exclude("org.apache.avro", "avro") + .exclude("org.apache.arrow", "arrow-vector") + .exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") +} + +/** + * Resolve the full Spark version for the given major/minor Spark + Scala versions from the + * `spark*-sql-scala*` dependency. + */ +fun Project.resolveFullSparkVersion( + sparkMajorMinorVersion: String, + scalaMajorMinorVersion: String, +): String { + val dotRegex = "[.]".toRegex() + val sparkVerTrimmed = sparkMajorMinorVersion.replace(dotRegex, "") + val scalaVerTrimmed = scalaMajorMinorVersion.replace(dotRegex, "") + return requiredDependencyPreferredVersion("spark$sparkVerTrimmed-sql-scala$scalaVerTrimmed") +} + +/** + * Resolve the full Scala version for the given major/minor Scala version from the + * `scala*-lang-library` dependency. + */ +fun Project.resolveFullScalaVersion(scalaMajorMinorVersion: String): String { + val dotRegex = "[.]".toRegex() + val scalaVerTrimmed = scalaMajorMinorVersion.replace(dotRegex, "") + return requiredDependencyPreferredVersion("scala$scalaVerTrimmed-lang-library") +} + +/** + * Get the Java LTS version, that is lower than or equal to the currently running Java version, for + * a given Spark version to be used in tests. + */ +fun javaVersionForSpark(sparkVersion: String): Int { + return when (sparkVersion) { + "3.4", + "3.5" -> 21 + else -> + throw IllegalArgumentException("Do not know which Java version Spark $sparkVersion supports") + } +} + +class SparkScalaVersions( + /** Spark major/minor version. */ + val sparkMajorMinorVersion: String, + /** Scala major/minor version. */ + val scalaMajorMinorVersion: String, + /** Full Spark version, including the patch version, for dependencies. */ + val sparkFullVersion: String, + /** Full Scala version, including the patch version, for dependencies. */ + val scalaFullVersion: String, + /** Java runtime version to be used in tests. */ + val runtimeJavaVersion: Int, +) + +/** + * Adds the JPMS options required for Spark to run on Java 17, taken from the + * `DEFAULT_MODULE_OPTIONS` constant in `org.apache.spark.launcher.JavaModuleOptions`. + */ +fun Project.addSparkJvmOptions() { + tasks.withType(Test::class.java).configureEach { + jvmArgs = + jvmArgs + + listOf( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED", + "-Djdk.reflect.useDirectMethodHandle=false", + // Required for Java 23+ if Hadoop stuff is used + "-Djava.security.manager=allow", + ) + } +} diff --git a/build-logic/src/main/kotlin/util.kt b/build-logic/src/main/kotlin/util.kt new file mode 100644 index 0000000000..971b82bc42 --- /dev/null +++ b/build-logic/src/main/kotlin/util.kt @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.lang.IllegalStateException +import org.gradle.api.JavaVersion +import org.gradle.api.Project +import org.gradle.api.artifacts.MinimalExternalModuleDependency +import org.gradle.api.artifacts.VersionCatalogsExtension +import org.gradle.api.invocation.Gradle +import org.gradle.api.tasks.testing.Test +import org.gradle.jvm.toolchain.JavaLanguageVersion +import org.gradle.jvm.toolchain.JavaToolchainService +import org.gradle.kotlin.dsl.getByType + +fun Gradle.ideSyncActive(): Boolean = + System.getProperty("idea.sync.active").toBoolean() || + System.getProperty("eclipse.product") != null || + startParameter.taskNames.any { it.startsWith("eclipse") } + +fun Project.requiredDependencyVersion(dependencyName: String): String = + requiredDependency(dependencyName).version!! + +fun Project.requiredDependencyPreferredVersion(dependencyName: String): String = + requiredDependency(dependencyName).versionConstraint.preferredVersion + +fun Project.requiredDependency(dependencyName: String): MinimalExternalModuleDependency { + val versionCatalog = extensions.getByType().named("libs") + val dependency = + versionCatalog.findLibrary(dependencyName).orElseThrow { + IllegalStateException("No library '$dependencyName' defined in version catalog 'libs'") + } + return dependency.get() +} + +/** Forces all [Test] tasks to use the given Java version. */ +fun Project.forceJavaVersionForTests(requestedJavaVersion: Int) { + if (requestedJavaVersion > 0) { + tasks.withType(Test::class.java).configureEach { + val currentJavaVersion = JavaVersion.current().majorVersion.toInt() + if (requestedJavaVersion != currentJavaVersion) { + useJavaVersion(requestedJavaVersion) + } + } + } +} + +fun Test.useJavaVersion(requestedJavaVersion: Int) { + val javaToolchains = project.extensions.findByType(JavaToolchainService::class.java) + logger.info("Configuring Java $requestedJavaVersion for $path test execution") + javaLauncher.set( + javaToolchains!!.launcherFor { + languageVersion.set(JavaLanguageVersion.of(requestedJavaVersion)) + } + ) +} diff --git a/codestyle/scalafmt.conf b/codestyle/scalafmt.conf new file mode 100644 index 0000000000..d02567b9ae --- /dev/null +++ b/codestyle/scalafmt.conf @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version = 3.9.4 +runner.dialect = scala213 + +maxColumn = 100 + +preset = default +align.preset = some + +assumeStandardLibraryStripMargin = true +align.stripMargin = true + +rewrite.rules = [ + AvoidInfix + RedundantBraces + RedundantParens + SortModifiers + PreferCurlyFors + Imports +] + +rewrite.imports.sort = original +docstrings.style = Asterisk +docstrings.wrap = fold diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4e569e27dd..f70ad6e28a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,8 +23,7 @@ iceberg = "1.7.1" quarkus = "3.19.3" immutables = "2.10.1" picocli = "4.7.6" -scala212 = "2.12.19" -spark35 = "3.5.5" +scalafmt = "3.9.4" slf4j = "2.0.17" swagger = "1.6.15" @@ -82,19 +81,29 @@ picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "pico postgresql = { module = "org.postgresql:postgresql", version = "42.7.5" } prometheus-metrics-exporter-servlet-jakarta = { module = "io.prometheus:prometheus-metrics-exporter-servlet-jakarta", version = "1.3.6" } quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkus" } -scala212-lang-library = { module = "org.scala-lang:scala-library", version.ref = "scala212" } -scala212-lang-reflect = { module = "org.scala-lang:scala-reflect", version.ref = "scala212" } +scalafmt = { module = "org.scalameta:scalafmt-core_2.13", version = "3.9.4" } s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", version = "3.12.0" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } smallrye-common-annotation = { module = "io.smallrye.common:smallrye-common-annotation", version = "2.10.0" } smallrye-config-core = { module = "io.smallrye.config:smallrye-config-core", version = "3.12.3" } -spark35-sql-scala212 = { module = "org.apache.spark:spark-sql_2.12", version.ref = "spark35" } spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version = "4.9.3" } swagger-annotations = { module = "io.swagger:swagger-annotations", version.ref = "swagger" } swagger-jaxrs = { module = "io.swagger:swagger-jaxrs", version.ref = "swagger" } testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version = "1.20.6" } threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" } +# Scala 2.12 +scala212-lang-library = { module = "org.scala-lang:scala-library", version = { strictly = "[2.12, 2.13[", prefer = "2.12.20" }} +scala212-lang-reflect = { module = "org.scala-lang:scala-reflect", version = { strictly = "[2.12, 2.13[", prefer = "2.12.20" }} +# Scala 2.13 +scala213-lang-library = { module = "org.scala-lang:scala-library", version = { strictly = "[2.13, 2.14[", prefer = "2.13.16" }} +scala213-lang-reflect = { module = "org.scala-lang:scala-reflect", version = { strictly = "[2.13, 2.14[", prefer = "2.13.16" }} +# Spark deps +spark34-sql-scala212 = { module = "org.apache.spark:spark-sql_2.12", version = { strictly = "[3.4, 3.5[", prefer = "3.4.4"}} +spark34-sql-scala213 = { module = "org.apache.spark:spark-sql_2.13", version = { strictly = "[3.4, 3.5[", prefer = "3.4.4"}} +spark35-sql-scala212 = { module = "org.apache.spark:spark-sql_2.12", version = { strictly = "[3.5, 3.6[", prefer = "3.5.5"}} +spark35-sql-scala213 = { module = "org.apache.spark:spark-sql_2.13", version = { strictly = "[3.5, 3.6[", prefer = "3.5.5"}} + [plugins] jandex = { id = "org.kordamp.gradle.jandex", version = "2.1.0" } openapi-generator = { id = "org.openapi.generator", version = "7.12.0" } diff --git a/integrations/demo/build.gradle.kts b/integrations/demo/build.gradle.kts new file mode 100644 index 0000000000..b7ba6ab695 --- /dev/null +++ b/integrations/demo/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { id("polaris-spark") } + +val scalaVersions = prepareScalaProject() diff --git a/integrations/demo/src/main/scala/org/apache/polaris/scalademo/SomeScala.scala b/integrations/demo/src/main/scala/org/apache/polaris/scalademo/SomeScala.scala new file mode 100644 index 0000000000..b27f81ff3e --- /dev/null +++ b/integrations/demo/src/main/scala/org/apache/polaris/scalademo/SomeScala.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.scalademo + +class SomeScala {} diff --git a/integrations/spark-scala.properties b/integrations/spark-scala.properties new file mode 100644 index 0000000000..49909cbae1 --- /dev/null +++ b/integrations/spark-scala.properties @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +sparkVersions=3.5,3.4 + +# Hint: the first Scala version in the list(s) is the one used in IDEs +sparkVersion-3.4-scalaVersions=2.12,2.13 +sparkVersion-3.5-scalaVersions=2.12,2.13 +# Scala versions for non-Spark Scala projects +allScalaVersions=2.12,2.13 + +# Spark+Scala projects (comma separated) +sparkProjects=spark +# Non-Spark Scala projects (comma separated) +scalaProjects=demo + +# Artifact IDs for the above projects +project.demo.artifact-id-prefix=polaris-scala-demo +project.spark.artifact-id-prefix=polaris-spark-plugin diff --git a/integrations/spark/build.gradle.kts b/integrations/spark/build.gradle.kts new file mode 100644 index 0000000000..18f631e04b --- /dev/null +++ b/integrations/spark/build.gradle.kts @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("polaris-spark") +} + +val sparkScala = prepareSparkScalaProject() + +dependencies { + compileOnly("org.apache.spark:spark-sql_${sparkScala.scalaMajorMinorVersion}") { sparkDependencyAndExcludes(sparkScala) } + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-spark-runtime-${sparkScala.sparkMajorMinorVersion}_${sparkScala.scalaMajorMinorVersion}") + + implementation(project(":polaris-scala-demo_${sparkScala.scalaMajorMinorVersion}")) + + compileOnly("org.apache.iceberg:iceberg-spark-${sparkScala.sparkMajorMinorVersion}_${sparkScala.scalaMajorMinorVersion}") + + testImplementation("org.apache.spark:spark-sql_${sparkScala.scalaMajorMinorVersion}") { sparkDependencyAndExcludes(sparkScala) } +} diff --git a/integrations/spark/v3.4/src/main/java/org/apache/polaris/sparkplugin/JavaStuff.java b/integrations/spark/v3.4/src/main/java/org/apache/polaris/sparkplugin/JavaStuff.java new file mode 100644 index 0000000000..018ad1cf90 --- /dev/null +++ b/integrations/spark/v3.4/src/main/java/org/apache/polaris/sparkplugin/JavaStuff.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin; + +import org.apache.polaris.scalademo.SomeScala; + +public class JavaStuff { + public SomeScala someScala = new SomeScala(); +} diff --git a/integrations/spark/v3.4/src/main/java/org/apache/polaris/sparkplugin/SparkCatalog.java b/integrations/spark/v3.4/src/main/java/org/apache/polaris/sparkplugin/SparkCatalog.java new file mode 100644 index 0000000000..c9329c5eb1 --- /dev/null +++ b/integrations/spark/v3.4/src/main/java/org/apache/polaris/sparkplugin/SparkCatalog.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin; + +import com.google.common.collect.ImmutableSet; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.spark.sql.catalyst.analysis.*; +import org.apache.spark.sql.connector.catalog.*; +import org.apache.spark.sql.connector.expressions.Transform; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +public class SparkCatalog implements TableCatalog, SupportsNamespaces, ViewCatalog { + private static final Set DEFAULT_NS_KEYS = ImmutableSet.of(TableCatalog.PROP_OWNER); + private String catalogName = null; + private org.apache.iceberg.spark.SparkCatalog icebergsSparkCatalog = null; + + // TODO: Add Polaris Specific REST Catalog + + @Override + public String name() { + return catalogName; + } + + @Override + public void initialize(String name, CaseInsensitiveStringMap options) { + this.catalogName = name; + } + + @Override + public Table loadTable(Identifier ident) throws NoSuchTableException { + throw new UnsupportedOperationException("loadTable"); + } + + @Override + public Table createTable( + Identifier ident, StructType schema, Transform[] transforms, Map properties) + throws TableAlreadyExistsException { + throw new UnsupportedOperationException("createTable"); + } + + @Override + public Table alterTable(Identifier ident, TableChange... changes) throws NoSuchTableException { + throw new UnsupportedOperationException("alterTable"); + } + + @Override + public boolean dropTable(Identifier ident) { + throw new UnsupportedOperationException("dropTable"); + } + + @Override + public void renameTable(Identifier from, Identifier to) + throws NoSuchTableException, TableAlreadyExistsException { + throw new UnsupportedOperationException("renameTable"); + } + + @Override + public Identifier[] listTables(String[] namespace) { + throw new UnsupportedOperationException("listTables"); + } + + @Override + public String[] defaultNamespace() { + throw new UnsupportedOperationException("defaultNamespace"); + } + + @Override + public String[][] listNamespaces() { + throw new UnsupportedOperationException("listNamespaces"); + } + + @Override + public String[][] listNamespaces(String[] namespace) throws NoSuchNamespaceException { + throw new UnsupportedOperationException("listNamespaces"); + } + + @Override + public Map loadNamespaceMetadata(String[] namespace) + throws NoSuchNamespaceException { + throw new UnsupportedOperationException("loadNamespaceMetadata"); + } + + @Override + public void createNamespace(String[] namespace, Map metadata) + throws NamespaceAlreadyExistsException { + throw new UnsupportedOperationException("createNamespace"); + } + + @Override + public void alterNamespace(String[] namespace, NamespaceChange... changes) + throws NoSuchNamespaceException { + throw new UnsupportedOperationException("alterNamespace"); + } + + @Override + public boolean dropNamespace(String[] namespace, boolean cascade) + throws NoSuchNamespaceException { + throw new UnsupportedOperationException("dropNamespace"); + } + + @Override + public Identifier[] listViews(String... namespace) { + throw new UnsupportedOperationException("listViews"); + } + + @Override + public View loadView(Identifier ident) throws NoSuchViewException { + throw new UnsupportedOperationException("loadView"); + } + + @Override + public View createView( + Identifier ident, + String sql, + String currentCatalog, + String[] currentNamespace, + StructType schema, + String[] queryColumnNames, + String[] columnAliases, + String[] columnComments, + Map properties) + throws ViewAlreadyExistsException, NoSuchNamespaceException { + throw new UnsupportedOperationException("createView"); + } + + @Override + public View alterView(Identifier ident, ViewChange... changes) + throws NoSuchViewException, IllegalArgumentException { + throw new UnsupportedOperationException("alterView"); + } + + @Override + public boolean dropView(Identifier ident) { + throw new UnsupportedOperationException("dropView"); + } + + @Override + public void renameView(Identifier fromIdentifier, Identifier toIdentifier) + throws NoSuchViewException, ViewAlreadyExistsException { + throw new UnsupportedOperationException("renameView"); + } +} diff --git a/integrations/spark/v3.4/src/main/scala/org/apache/polaris/sparkplugin/ScalaStuff.scala b/integrations/spark/v3.4/src/main/scala/org/apache/polaris/sparkplugin/ScalaStuff.scala new file mode 100644 index 0000000000..e3af3ebba0 --- /dev/null +++ b/integrations/spark/v3.4/src/main/scala/org/apache/polaris/sparkplugin/ScalaStuff.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin + +import org.apache.polaris.scalademo.SomeScala + +class ScalaStuff { + new SomeScala() +} diff --git a/integrations/spark/v3.4/src/test/java/org/apache/polaris/sparkplugin/SparkCatalogTest.java b/integrations/spark/v3.4/src/test/java/org/apache/polaris/sparkplugin/SparkCatalogTest.java new file mode 100644 index 0000000000..57666468f8 --- /dev/null +++ b/integrations/spark/v3.4/src/test/java/org/apache/polaris/sparkplugin/SparkCatalogTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin; + +import static org.apache.iceberg.CatalogProperties.CATALOG_IMPL; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Map; +import java.util.UUID; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class SparkCatalogTest { + private SparkCatalog catalog; + private String catalogName; + + @BeforeEach + public void setup() { + catalogName = "test_" + UUID.randomUUID(); + Map catalogConfig = Maps.newHashMap(); + catalogConfig.put(CATALOG_IMPL, "org.apache.iceberg.inmemory.InMemoryCatalog"); + catalogConfig.put("cache-enabled", "false"); + catalog = new SparkCatalog(); + catalog.initialize(catalogName, new CaseInsensitiveStringMap(catalogConfig)); + } + + @Test + public void testUnsupportedOperations() { + String[] namespace = new String[] {"ns1"}; + Identifier identifier = Identifier.of(namespace, "table1"); + Identifier new_identifier = Identifier.of(namespace, "table2"); + // table methods + assertThatThrownBy(() -> catalog.loadTable(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy( + () -> catalog.createTable(identifier, Mockito.mock(StructType.class), null, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.alterTable(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.dropTable(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.renameTable(identifier, new_identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.listTables(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + + // namespace methods + assertThatThrownBy(() -> catalog.loadNamespaceMetadata(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.listNamespaces()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.listNamespaces(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.createNamespace(namespace, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.alterNamespace(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.dropNamespace(namespace, false)) + .isInstanceOf(UnsupportedOperationException.class); + + // view methods + assertThatThrownBy(() -> catalog.listViews(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.loadView(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy( + () -> catalog.createView(identifier, null, null, null, null, null, null, null, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.alterView(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.dropView(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.renameView(identifier, new_identifier)) + .isInstanceOf(UnsupportedOperationException.class); + } +} diff --git a/integrations/spark/v3.4/src/test/java/org/apache/polaris/sparkplugin/TestStuff.java b/integrations/spark/v3.4/src/test/java/org/apache/polaris/sparkplugin/TestStuff.java new file mode 100644 index 0000000000..ffee405ae4 --- /dev/null +++ b/integrations/spark/v3.4/src/test/java/org/apache/polaris/sparkplugin/TestStuff.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin; + +import org.junit.jupiter.api.Test; + +public class TestStuff { + @Test + public void javaStuff() { + new JavaStuff(); + } + + @Test + public void scalaStuff() { + new ScalaStuff(); + } +} diff --git a/integrations/spark/v3.5/src/main/java/org/apache/polaris/sparkplugin/JavaStuff.java b/integrations/spark/v3.5/src/main/java/org/apache/polaris/sparkplugin/JavaStuff.java new file mode 100644 index 0000000000..018ad1cf90 --- /dev/null +++ b/integrations/spark/v3.5/src/main/java/org/apache/polaris/sparkplugin/JavaStuff.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin; + +import org.apache.polaris.scalademo.SomeScala; + +public class JavaStuff { + public SomeScala someScala = new SomeScala(); +} diff --git a/integrations/spark/v3.5/src/main/java/org/apache/polaris/sparkplugin/SparkCatalog.java b/integrations/spark/v3.5/src/main/java/org/apache/polaris/sparkplugin/SparkCatalog.java new file mode 100644 index 0000000000..c9329c5eb1 --- /dev/null +++ b/integrations/spark/v3.5/src/main/java/org/apache/polaris/sparkplugin/SparkCatalog.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin; + +import com.google.common.collect.ImmutableSet; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.spark.sql.catalyst.analysis.*; +import org.apache.spark.sql.connector.catalog.*; +import org.apache.spark.sql.connector.expressions.Transform; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +public class SparkCatalog implements TableCatalog, SupportsNamespaces, ViewCatalog { + private static final Set DEFAULT_NS_KEYS = ImmutableSet.of(TableCatalog.PROP_OWNER); + private String catalogName = null; + private org.apache.iceberg.spark.SparkCatalog icebergsSparkCatalog = null; + + // TODO: Add Polaris Specific REST Catalog + + @Override + public String name() { + return catalogName; + } + + @Override + public void initialize(String name, CaseInsensitiveStringMap options) { + this.catalogName = name; + } + + @Override + public Table loadTable(Identifier ident) throws NoSuchTableException { + throw new UnsupportedOperationException("loadTable"); + } + + @Override + public Table createTable( + Identifier ident, StructType schema, Transform[] transforms, Map properties) + throws TableAlreadyExistsException { + throw new UnsupportedOperationException("createTable"); + } + + @Override + public Table alterTable(Identifier ident, TableChange... changes) throws NoSuchTableException { + throw new UnsupportedOperationException("alterTable"); + } + + @Override + public boolean dropTable(Identifier ident) { + throw new UnsupportedOperationException("dropTable"); + } + + @Override + public void renameTable(Identifier from, Identifier to) + throws NoSuchTableException, TableAlreadyExistsException { + throw new UnsupportedOperationException("renameTable"); + } + + @Override + public Identifier[] listTables(String[] namespace) { + throw new UnsupportedOperationException("listTables"); + } + + @Override + public String[] defaultNamespace() { + throw new UnsupportedOperationException("defaultNamespace"); + } + + @Override + public String[][] listNamespaces() { + throw new UnsupportedOperationException("listNamespaces"); + } + + @Override + public String[][] listNamespaces(String[] namespace) throws NoSuchNamespaceException { + throw new UnsupportedOperationException("listNamespaces"); + } + + @Override + public Map loadNamespaceMetadata(String[] namespace) + throws NoSuchNamespaceException { + throw new UnsupportedOperationException("loadNamespaceMetadata"); + } + + @Override + public void createNamespace(String[] namespace, Map metadata) + throws NamespaceAlreadyExistsException { + throw new UnsupportedOperationException("createNamespace"); + } + + @Override + public void alterNamespace(String[] namespace, NamespaceChange... changes) + throws NoSuchNamespaceException { + throw new UnsupportedOperationException("alterNamespace"); + } + + @Override + public boolean dropNamespace(String[] namespace, boolean cascade) + throws NoSuchNamespaceException { + throw new UnsupportedOperationException("dropNamespace"); + } + + @Override + public Identifier[] listViews(String... namespace) { + throw new UnsupportedOperationException("listViews"); + } + + @Override + public View loadView(Identifier ident) throws NoSuchViewException { + throw new UnsupportedOperationException("loadView"); + } + + @Override + public View createView( + Identifier ident, + String sql, + String currentCatalog, + String[] currentNamespace, + StructType schema, + String[] queryColumnNames, + String[] columnAliases, + String[] columnComments, + Map properties) + throws ViewAlreadyExistsException, NoSuchNamespaceException { + throw new UnsupportedOperationException("createView"); + } + + @Override + public View alterView(Identifier ident, ViewChange... changes) + throws NoSuchViewException, IllegalArgumentException { + throw new UnsupportedOperationException("alterView"); + } + + @Override + public boolean dropView(Identifier ident) { + throw new UnsupportedOperationException("dropView"); + } + + @Override + public void renameView(Identifier fromIdentifier, Identifier toIdentifier) + throws NoSuchViewException, ViewAlreadyExistsException { + throw new UnsupportedOperationException("renameView"); + } +} diff --git a/integrations/spark/v3.5/src/main/scala/org/apache/polaris/sparkplugin/ScalaStuff.scala b/integrations/spark/v3.5/src/main/scala/org/apache/polaris/sparkplugin/ScalaStuff.scala new file mode 100644 index 0000000000..e3af3ebba0 --- /dev/null +++ b/integrations/spark/v3.5/src/main/scala/org/apache/polaris/sparkplugin/ScalaStuff.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin + +import org.apache.polaris.scalademo.SomeScala + +class ScalaStuff { + new SomeScala() +} diff --git a/integrations/spark/v3.5/src/test/java/org/apache/polaris/sparkplugin/SparkCatalogTest.java b/integrations/spark/v3.5/src/test/java/org/apache/polaris/sparkplugin/SparkCatalogTest.java new file mode 100644 index 0000000000..57666468f8 --- /dev/null +++ b/integrations/spark/v3.5/src/test/java/org/apache/polaris/sparkplugin/SparkCatalogTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin; + +import static org.apache.iceberg.CatalogProperties.CATALOG_IMPL; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Map; +import java.util.UUID; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class SparkCatalogTest { + private SparkCatalog catalog; + private String catalogName; + + @BeforeEach + public void setup() { + catalogName = "test_" + UUID.randomUUID(); + Map catalogConfig = Maps.newHashMap(); + catalogConfig.put(CATALOG_IMPL, "org.apache.iceberg.inmemory.InMemoryCatalog"); + catalogConfig.put("cache-enabled", "false"); + catalog = new SparkCatalog(); + catalog.initialize(catalogName, new CaseInsensitiveStringMap(catalogConfig)); + } + + @Test + public void testUnsupportedOperations() { + String[] namespace = new String[] {"ns1"}; + Identifier identifier = Identifier.of(namespace, "table1"); + Identifier new_identifier = Identifier.of(namespace, "table2"); + // table methods + assertThatThrownBy(() -> catalog.loadTable(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy( + () -> catalog.createTable(identifier, Mockito.mock(StructType.class), null, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.alterTable(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.dropTable(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.renameTable(identifier, new_identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.listTables(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + + // namespace methods + assertThatThrownBy(() -> catalog.loadNamespaceMetadata(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.listNamespaces()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.listNamespaces(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.createNamespace(namespace, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.alterNamespace(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.dropNamespace(namespace, false)) + .isInstanceOf(UnsupportedOperationException.class); + + // view methods + assertThatThrownBy(() -> catalog.listViews(namespace)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.loadView(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy( + () -> catalog.createView(identifier, null, null, null, null, null, null, null, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.alterView(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.dropView(identifier)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> catalog.renameView(identifier, new_identifier)) + .isInstanceOf(UnsupportedOperationException.class); + } +} diff --git a/integrations/spark/v3.5/src/test/java/org/apache/polaris/sparkplugin/TestStuff.java b/integrations/spark/v3.5/src/test/java/org/apache/polaris/sparkplugin/TestStuff.java new file mode 100644 index 0000000000..ffee405ae4 --- /dev/null +++ b/integrations/spark/v3.5/src/test/java/org/apache/polaris/sparkplugin/TestStuff.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.sparkplugin; + +import org.junit.jupiter.api.Test; + +public class TestStuff { + @Test + public void javaStuff() { + new JavaStuff(); + } + + @Test + public void scalaStuff() { + new ScalaStuff(); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 707592ead1..04d04fff80 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -44,19 +44,82 @@ fun loadProperties(file: File): Properties { return props } -fun polarisProject(name: String, directory: File) { +fun listFromProperty(props: Properties, key: String): List { + val prop = props[key] + return prop.toString().split(',').filter { it.isNotBlank() }.map { it.trim() } +} + +fun polarisProject(name: String, directory: File): ProjectDescriptor { include(name) val prj = project(":${name}") prj.name = name prj.projectDir = file(directory) + return prj } -val projects = Properties() - loadProperties(file("gradle/projects.main.properties")).forEach { name, directory -> polarisProject(name as String, file(directory as String)) } +val ideSyncActive = + System.getProperty("idea.sync.active").toBoolean() || + System.getProperty("eclipse.product") != null || + gradle.startParameter.taskNames.any { it.startsWith("eclipse") } + +// Setup Spark/Scala projects + +val projectsWithReusedSourceDirs = mutableSetOf() +val sparkScala = loadProperties(file("integrations/spark-scala.properties")) + +val sparkVersions = listFromProperty(sparkScala, "sparkVersions") + +for (sparkVersion in sparkVersions) { + val scalaVersions = + sparkScala["sparkVersion-${sparkVersion}-scalaVersions"].toString().split(",").map { it.trim() } + var first = true + for (scalaVersion in scalaVersions) { + for (prj in listFromProperty(sparkScala, "sparkProjects")) { + val artifactIdPrefix = sparkScala["project.$prj.artifact-id-prefix"].toString() + val artifactId = "$artifactIdPrefix-${sparkVersion}_$scalaVersion" + if (first) { + first = false + } else { + projectsWithReusedSourceDirs.add(":$artifactId") + } + polarisProject(artifactId, file("integrations/$prj/v${sparkVersion}")).buildFileName = + "../build.gradle.kts" + } + if (ideSyncActive) { + break + } + } +} + +// Setup Spark/Scala and "pure" Scala projects + +val allScalaVersions = listFromProperty(sparkScala, "allScalaVersions") + +var first = true + +for (scalaVersion in allScalaVersions) { + for (prj in listFromProperty(sparkScala, "scalaProjects")) { + val name = sparkScala["project.$prj.artifact-id-prefix"].toString() + val artifactId = "${name}_$scalaVersion" + if (!first) { + projectsWithReusedSourceDirs.add(":$artifactId") + } + polarisProject(artifactId, file("integrations/$prj")) + } + if (first) { + first = false + } + if (ideSyncActive) { + break + } +} + +// Plugin, dependency management, etc + pluginManagement { repositories { mavenCentral() // prefer Maven Central, in case Gradle's repo has issues @@ -75,4 +138,9 @@ dependencyResolutionManagement { gradle.beforeProject { version = baseVersion group = "org.apache.polaris" + + // Mark projects that reuse source directories, so those do not re-run spotless, errorprone, etc + if (projectsWithReusedSourceDirs.contains(this.path)) { + project.extra["reused-project-dir"] = true + } }