diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index badfd0dfafb63..73d08eaa2b16f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -767,7 +767,7 @@ object CatalogTable { props.get(key).orElse { if (props.exists { case (mapKey, _) => mapKey.startsWith(key) }) { props.get(s"$key.numParts") match { - case None => throw QueryCompilationErrors.insufficientTablePropertyError(key) + case None => None case Some(numParts) => val parts = (0 until numParts.toInt).map { index => val keyPart = s"$key.part.$index" diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 9244776a20088..772db8dff615f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -1378,62 +1378,78 @@ class MetastoreDataSourcesSuite extends QueryTest } test("read table with corrupted schema") { - try { - val schema = StructType(StructField("int", IntegerType) :: Nil) - val hiveTableWithoutNumPartsProp = CatalogTable( - identifier = TableIdentifier("t", Some("default")), - tableType = CatalogTableType.MANAGED, - schema = HiveExternalCatalog.EMPTY_DATA_SCHEMA, - provider = Some("json"), - storage = CatalogStorageFormat.empty, - properties = Map( - DATASOURCE_PROVIDER -> "json", - DATASOURCE_SCHEMA_PART_PREFIX + 0 -> schema.json)) - - hiveClient.createTable(hiveTableWithoutNumPartsProp, ignoreIfExists = false) - - checkError( - exception = intercept[AnalysisException] { - sharedState.externalCatalog.getTable("default", "t") - }, - condition = "INSUFFICIENT_TABLE_PROPERTY.MISSING_KEY", - parameters = Map("key" -> toSQLConf("spark.sql.sources.schema")) - ) - - val hiveTableWithNumPartsProp = CatalogTable( - identifier = TableIdentifier("t2", Some("default")), - tableType = CatalogTableType.MANAGED, - schema = HiveExternalCatalog.EMPTY_DATA_SCHEMA, - provider = Some("json"), - storage = CatalogStorageFormat.empty, - properties = Map( - DATASOURCE_PROVIDER -> "json", - DATASOURCE_SCHEMA_PREFIX + "numParts" -> "3", - DATASOURCE_SCHEMA_PART_PREFIX + 0 -> schema.json)) - - hiveClient.createTable(hiveTableWithNumPartsProp, ignoreIfExists = false) + Seq(true, false).foreach { isHiveTable => + try { + val schema = StructType(StructField("int", IntegerType) :: Nil) + val hiveTableWithoutNumPartsProp = CatalogTable( + identifier = TableIdentifier("t", Some("default")), + tableType = CatalogTableType.MANAGED, + schema = HiveExternalCatalog.EMPTY_DATA_SCHEMA, + provider = if (isHiveTable) None else Some("json"), + storage = CatalogStorageFormat.empty, + properties = Map( + DATASOURCE_SCHEMA_PART_PREFIX + 0 -> schema.json) ++ { + if (isHiveTable) { + Map.empty + } else { + Map(DATASOURCE_PROVIDER -> "json") + } + }) - checkError( - exception = intercept[AnalysisException] { - sharedState.externalCatalog.getTable("default", "t2") - }, - condition = "INSUFFICIENT_TABLE_PROPERTY.MISSING_KEY_PART", - parameters = Map( - "key" -> toSQLConf("spark.sql.sources.schema.part.1"), - "totalAmountOfParts" -> "3") - ) + hiveClient.createTable(hiveTableWithoutNumPartsProp, ignoreIfExists = false) - withDebugMode { val tableMeta = sharedState.externalCatalog.getTable("default", "t") assert(tableMeta.identifier == TableIdentifier("t", Some("default"))) - assert(tableMeta.properties(DATASOURCE_PROVIDER) == "json") - val tableMeta2 = sharedState.externalCatalog.getTable("default", "t2") - assert(tableMeta2.identifier == TableIdentifier("t2", Some("default"))) - assert(tableMeta2.properties(DATASOURCE_PROVIDER) == "json") + assert(!tableMeta.properties.contains(DATASOURCE_PROVIDER)) + + val hiveTableWithNumPartsProp = CatalogTable( + identifier = TableIdentifier("t2", Some("default")), + tableType = CatalogTableType.MANAGED, + schema = HiveExternalCatalog.EMPTY_DATA_SCHEMA, + provider = if (isHiveTable) None else Some("json"), + storage = CatalogStorageFormat.empty, + properties = Map( + DATASOURCE_SCHEMA_PREFIX + "numParts" -> "3", + DATASOURCE_SCHEMA_PART_PREFIX + 0 -> schema.json) ++ { + if (isHiveTable) { + Map.empty + } else { + Map(DATASOURCE_PROVIDER -> "json") + } + }) + + hiveClient.createTable(hiveTableWithNumPartsProp, ignoreIfExists = false) + + checkError( + exception = intercept[AnalysisException] { + sharedState.externalCatalog.getTable("default", "t2") + }, + condition = "INSUFFICIENT_TABLE_PROPERTY.MISSING_KEY_PART", + parameters = Map( + "key" -> toSQLConf("spark.sql.sources.schema.part.1"), + "totalAmountOfParts" -> "3") + ) + + withDebugMode { + val tableMeta = sharedState.externalCatalog.getTable("default", "t") + assert(tableMeta.identifier == TableIdentifier("t", Some("default"))) + if (isHiveTable) { + assert(!tableMeta.properties.contains(DATASOURCE_PROVIDER)) + } else { + assert(tableMeta.properties(DATASOURCE_PROVIDER) == "json") + } + val tableMeta2 = sharedState.externalCatalog.getTable("default", "t2") + assert(tableMeta2.identifier == TableIdentifier("t2", Some("default"))) + if (isHiveTable) { + assert(!tableMeta2.properties.contains(DATASOURCE_PROVIDER)) + } else { + assert(tableMeta2.properties(DATASOURCE_PROVIDER) == "json") + } + } + } finally { + hiveClient.dropTable("default", "t", ignoreIfNotExists = true, purge = true) + hiveClient.dropTable("default", "t2", ignoreIfNotExists = true, purge = true) } - } finally { - hiveClient.dropTable("default", "t", ignoreIfNotExists = true, purge = true) - hiveClient.dropTable("default", "t2", ignoreIfNotExists = true, purge = true) } }