Skip to content

[SPARK-37929][SQL][FOLLOWUP] Support cascade mode for JDBC V2 #35271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
beliefer wants to merge 5 commits into from
Closed

[SPARK-37929][SQL][FOLLOWUP] Support cascade mode for JDBC V2 #35271

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2a2f881
[SPARK-37929][SQL] Support cascade mode for JDBC V2
beliefer Jan 21, 2022
146ac41
Update code
beliefer Jan 22, 2022
4ebd3f3
Update code
beliefer Jan 22, 2022
0744168
Update code
beliefer Jan 25, 2022
6bc248d
Update code
beliefer Jan 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion ...r-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,30 @@

package org.apache.spark.sql.jdbc.v2

import java.util
import java.util.Collections

import scala.collection.JavaConverters._

import org.apache.logging.log4j.Level

import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.connector.catalog.NamespaceChange
import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange}
import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
import org.apache.spark.sql.jdbc.DockerIntegrationFunSuite
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
import org.apache.spark.tags.DockerTest

@DockerTest
private[v2] trait V2JDBCNamespaceTest extends SharedSparkSession with DockerIntegrationFunSuite {
val catalog = new JDBCTableCatalog()

private val emptyProps: util.Map[String, String] = Collections.emptyMap[String, String]
private val schema: StructType = new StructType()
.add("id", IntegerType)
.add("data", StringType)

def builtinNamespaces: Array[Array[String]]

test("listNamespaces: basic behavior") {
Expand Down Expand Up @@ -60,4 +69,27 @@ private[v2] trait V2JDBCNamespaceTest extends SharedSparkSession with DockerInte
}.getMessage
assert(msg.contains("Namespace 'foo' not found"))
}

test("Drop namespace") {
val ident1 = Identifier.of(Array("foo"), "tab")
// Drop empty namespace without cascade
catalog.createNamespace(Array("foo"), Map("comment" -> "test comment").asJava)
assert(catalog.namespaceExists(Array("foo")) === true)
catalog.dropNamespace(Array("foo"), cascade = false)
assert(catalog.namespaceExists(Array("foo")) === false)

// Drop non empty namespace without cascade
catalog.createNamespace(Array("foo"), Map("comment" -> "test comment").asJava)
assert(catalog.namespaceExists(Array("foo")) === true)
catalog.createTable(ident1, schema, Array.empty, emptyProps)
val msg = intercept[IllegalStateException] {
catalog.dropNamespace(Array("foo"), cascade = false)
}.getMessage
assert(msg.contains("Namespace foo is not empty"))

// Drop non empty namespace with cascade
assert(catalog.namespaceExists(Array("foo")) === true)
catalog.dropNamespace(Array("foo"), cascade = true)
assert(catalog.namespaceExists(Array("foo")) === false)
}
}
10 changes: 8 additions & 2 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1014,9 +1014,15 @@ object JdbcUtils extends Logging with SQLConfHelper {
/**
* Drops a namespace from the JDBC database.
*/
def dropNamespace(conn: Connection, options: JDBCOptions, namespace: String): Unit = {
def dropNamespace(
conn: Connection, options: JDBCOptions, namespace: String, cascade: Boolean): Unit = {
val dialect = JdbcDialects.get(options.url)
executeStatement(conn, options, s"DROP SCHEMA ${dialect.quoteIdentifier(namespace)}")
val dropCmd = if (cascade) {
s"DROP SCHEMA ${dialect.quoteIdentifier(namespace)} CASCADE"
} else {
s"DROP SCHEMA ${dialect.quoteIdentifier(namespace)}"
}
executeStatement(conn, options, dropCmd)
}

/**
Expand Down
4 changes: 2 additions & 2 deletions .../src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,12 +282,12 @@ class JDBCTableCatalog extends TableCatalog with SupportsNamespaces with Logging
namespace: Array[String],
cascade: Boolean): Boolean = namespace match {
case Array(db) if namespaceExists(namespace) =>
if (listTables(Array(db)).nonEmpty) {
if (!cascade && listTables(Array(db)).nonEmpty) {
Copy link
Contributor

@cloud-fan cloud-fan Jan 25, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

listTables could be expensive. Can we try-catch the exception thrown by JDBC and detect the case when the schema/database is not empty?

Copy link
Contributor Author

@beliefer beliefer Jan 25, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK

throw QueryExecutionErrors.namespaceNotEmptyError(namespace)
}
JdbcUtils.withConnection(options) { conn =>
JdbcUtils.classifyException(s"Failed drop name space: $db", dialect) {
JdbcUtils.dropNamespace(conn, options, db)
JdbcUtils.dropNamespace(conn, options, db, cascade)
true
}
}
Expand Down