diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala index 66ac9ddb21aa..3f10e2d88afe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala @@ -36,9 +36,12 @@ object StaticSQLConf { .createWithDefault(Utils.resolveURI("spark-warehouse").toString) val CATALOG_IMPLEMENTATION = buildStaticConf("spark.sql.catalogImplementation") - .internal() + .doc("Spark built-in implementation is in-memory and hive, " + + "when set to yourImpl(can be any words), " + + "you need also provide following configurations to make everything going: " + + "`spark.sql.catalogImplementation.yourImpl.builder` " + + "`spark.sql.catalogImplementation.yourImpl.externalCatalog`.") .stringConf - .checkValues(Set("hive", "in-memory")) .createWithDefault("in-memory") val GLOBAL_TEMP_DATABASE = buildStaticConf("spark.sql.globalTempDatabase") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index bd2bc1c0ad5d..aa603f2faed2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -1026,6 +1026,12 @@ object SparkSession extends Logging { conf.get(CATALOG_IMPLEMENTATION) match { case "hive" => HIVE_SESSION_STATE_BUILDER_CLASS_NAME case "in-memory" => classOf[SessionStateBuilder].getCanonicalName + case other => conf.getOption(s"spark.sql.catalogImplementation.$other.builder") + .getOrElse { + throw new IllegalArgumentException( + s"You need to configure spark.sql.catalogImplementation.$other.builder when " + + s"`$other` configured by spark.sql.catalogImplementation is not in-memory nor hive.") + } } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index de3805e10580..08e3a0c15c68 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -217,6 +217,12 @@ object SharedState extends Logging { conf.get(CATALOG_IMPLEMENTATION) match { case "hive" => HIVE_EXTERNAL_CATALOG_CLASS_NAME case "in-memory" => classOf[InMemoryCatalog].getCanonicalName + case other => conf.getOption(s"spark.sql.catalogImplementation.$other.externalCatalog") + .getOrElse { + throw new IllegalArgumentException( + s"You need to configure spark.sql.catalogImplementation.$other.externalCatalog when " + + s"`$other` configured by spark.sql.catalogImplementation is not in-memory nor hive.") + } } }