From a74348280e82f5a47f7ae4f3909b61aeee8f6de8 Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Wed, 9 Feb 2022 20:37:35 +0800 Subject: [PATCH 1/7] Determine if table path is absolute by checking its first letter --- .../sql/catalyst/catalog/SessionCatalog.scala | 2 ++ .../v2/V2SessionCatalogSuite.scala | 25 ++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 464768ac7ce2b..1c30f10f63d38 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -388,6 +388,8 @@ class SessionCatalog( private def makeQualifiedTablePath(locationUri: URI, database: String): URI = { if (locationUri.isAbsolute) { locationUri + } else if (locationUri.getPath.startsWith("/")) { + makeQualifiedPath(locationUri) } else { val dbName = formatDatabaseName(database) val dbLocation = makeQualifiedDBPath(getDatabaseMetadata(dbName).locationUri) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala index 1aa8e3736cfe2..bae793bb01214 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala @@ -29,7 +29,7 @@ import org.scalatest.BeforeAndAfter import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.{NamespaceAlreadyExistsException, NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException, TableAlreadyExistsException} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, NamespaceChange, TableCatalog, TableChange, V1Table} +import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, NamespaceChange, SupportsNamespaces, TableCatalog, TableChange, V1Table} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{DoubleType, IntegerType, LongType, StringType, StructField, StructType, TimestampType} import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -60,7 +60,8 @@ class V2SessionCatalogTableSuite extends V2SessionCatalogBaseSuite { super.beforeAll() val catalog = newCatalog() catalog.createNamespace(Array("db"), emptyProps) - catalog.createNamespace(Array("db2"), emptyProps) + catalog.createNamespace(Array("db2"), + Map(SupportsNamespaces.PROP_LOCATION -> "file:///db2.db").asJava) catalog.createNamespace(Array("ns"), emptyProps) catalog.createNamespace(Array("ns2"), emptyProps) } @@ -186,10 +187,17 @@ class V2SessionCatalogTableSuite extends V2SessionCatalogBaseSuite { assert(t2.catalogTable.location === makeQualifiedPathWithWarehouse("db.db/relative/path")) catalog.dropTable(testIdent) - // absolute path + // absolute path without scheme properties.put(TableCatalog.PROP_LOCATION, "/absolute/path") val t3 = catalog.createTable(testIdent, schema, Array.empty, properties).asInstanceOf[V1Table] - assert(t3.catalogTable.location.toString === "file:/absolute/path") + assert(t3.catalogTable.location.toString === "file:///absolute/path") + catalog.dropTable(testIdent) + + // absolute path with scheme + properties.put(TableCatalog.PROP_LOCATION, "file:/absolute/path") + val t4 = catalog.createTable(testIdent, schema, Array.empty, properties).asInstanceOf[V1Table] + assert(t4.catalogTable.location.toString === "file:/absolute/path") + catalog.dropTable(testIdent) } test("tableExists") { @@ -685,10 +693,15 @@ class V2SessionCatalogTableSuite extends V2SessionCatalogBaseSuite { TableChange.setProperty(TableCatalog.PROP_LOCATION, "relative/path")).asInstanceOf[V1Table] assert(t2.catalogTable.location === makeQualifiedPathWithWarehouse("db.db/relative/path")) - // absolute path + // absolute path without scheme val t3 = catalog.alterTable(testIdent, TableChange.setProperty(TableCatalog.PROP_LOCATION, "/absolute/path")).asInstanceOf[V1Table] - assert(t3.catalogTable.location.toString === "file:/absolute/path") + assert(t3.catalogTable.location.toString === "file:///absolute/path") + + // absolute path with scheme + val t4 = catalog.alterTable(testIdent, TableChange.setProperty( + TableCatalog.PROP_LOCATION, "file:/absolute/path")).asInstanceOf[V1Table] + assert(t4.catalogTable.location.toString === "file:/absolute/path") } test("dropTable") { From 67c23181d24a0b2e307025122a901af105133d16 Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Fri, 11 Feb 2022 13:59:55 +0800 Subject: [PATCH 2/7] Split out CommonFileDataSourceSuite --- .../org/apache/spark/sql/avro/AvroSuite.scala | 29 ++++++++++++++--- .../source/libsvm/LibSVMRelationSuite.scala | 32 +++++++++++-------- .../results/show-create-table.sql.out | 4 +-- .../sql/connector/DataSourceV2SQLSuite.scala | 2 +- .../command/v2/ShowCreateTableSuite.scala | 2 +- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index d85baeb9386f2..9172a9aa0db74 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -53,15 +53,11 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.v2.avro.AvroScan import org.apache.spark.util.Utils -abstract class AvroSuite +abstract class AvroSuiteBase extends QueryTest with SharedSparkSession - with CommonFileDataSourceSuite with NestedDataSourceSuiteBase { - import testImplicits._ - - override protected def dataSourceFormat = "avro" override val nestedDataSources = Seq("avro") val episodesAvro = testFile("episodes.avro") val testAvro = testFile("test.avro") @@ -95,6 +91,11 @@ abstract class AvroSuite } }, new GenericDatumReader[Any]()).getSchema.toString(false) } +} + +abstract class AvroSuite extends AvroSuiteBase { + + import testImplicits._ private def getResourceAvroFilePath(name: String): String = { Thread.currentThread().getContextClassLoader.getResource(name).toString @@ -2434,3 +2435,21 @@ class AvroV2Suite extends AvroSuite with ExplainSuiteHelper { } } } + +class AvroV1SuiteWithCommonFileSourceCheck extends AvroSuiteBase with CommonFileDataSourceSuite { + override protected def sparkConf: SparkConf = + super + .sparkConf + .set(SQLConf.USE_V1_SOURCE_LIST, "avro") + + override protected def dataSourceFormat = "avro" +} + +class AvroV2SuiteWithCommonFileSourceCheck extends AvroSuiteBase with CommonFileDataSourceSuite { + override protected def sparkConf: SparkConf = + super + .sparkConf + .set(SQLConf.USE_V1_SOURCE_LIST, "") + + override protected def dataSourceFormat = "avro" +} diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala index a456409cfe3bc..d0191ed9b06ca 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala @@ -32,20 +32,9 @@ import org.apache.spark.sql.execution.datasources.CommonFileDataSourceSuite import org.apache.spark.sql.types.{DoubleType, StructField, StructType} import org.apache.spark.util.Utils -class LibSVMRelationSuite +abstract class LibSVMRelationSuiteBase extends SparkFunSuite - with MLlibTestSparkContext - with CommonFileDataSourceSuite { - - override protected def dataSourceFormat = "libsvm" - override protected def inputDataset = { - val rawData = new java.util.ArrayList[Row]() - rawData.add(Row(1.0, Vectors.sparse(1, Seq((0, 1.0))))) - val struct = new StructType() - .add("labelFoo", DoubleType, false) - .add("featuresBar", VectorType, false) - spark.createDataFrame(rawData, struct) - } + with MLlibTestSparkContext { // Path for dataset var path: String = _ @@ -78,6 +67,9 @@ class LibSVMRelationSuite super.afterAll() } } +} + +class LibSVMRelationSuite extends LibSVMRelationSuiteBase { test("select as sparse vector") { val df = spark.read.format("libsvm").load(path) @@ -226,3 +218,17 @@ class LibSVMRelationSuite } } } + +class LibSVMRelationSuiteWithCommonFileSourceCheck + extends LibSVMRelationSuiteBase with CommonFileDataSourceSuite { + + override protected def dataSourceFormat = "libsvm" + override protected def inputDataset = { + val rawData = new java.util.ArrayList[Row]() + rawData.add(Row(1.0, Vectors.sparse(1, Seq((0, 1.0))))) + val struct = new StructType() + .add("labelFoo", DoubleType, false) + .add("featuresBar", VectorType, false) + spark.createDataFrame(rawData, struct) + } +} diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out index ca1652b337a28..ded27abc4c14d 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out @@ -80,7 +80,7 @@ CREATE TABLE default.tbl ( b STRING, c INT) USING parquet -LOCATION 'file:/path/to/table' +LOCATION 'file:///path/to/table' -- !query @@ -110,7 +110,7 @@ CREATE TABLE default.tbl ( b STRING, c INT) USING parquet -LOCATION 'file:/path/to/table' +LOCATION 'file:///path/to/table' -- !query diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index d9e3342240bcf..b64ed080d8bf1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -2773,7 +2773,7 @@ class DataSourceV2SQLSuite val properties = table.properties assert(properties.get(TableCatalog.PROP_PROVIDER) == "parquet") assert(properties.get(TableCatalog.PROP_COMMENT) == "This is a comment") - assert(properties.get(TableCatalog.PROP_LOCATION) == "file:/tmp") + assert(properties.get(TableCatalog.PROP_LOCATION) == "file:///tmp") assert(properties.containsKey(TableCatalog.PROP_OWNER)) assert(properties.get(TableCatalog.PROP_EXTERNAL) == "true") assert(properties.get(s"${TableCatalog.OPTION_PREFIX}from") == "0") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala index 7c506812079ec..2adbf3cc1c8c0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala @@ -101,7 +101,7 @@ class ShowCreateTableSuite extends command.ShowCreateTableSuiteBase with Command "'via' = '2')", "PARTITIONED BY (a)", "COMMENT 'This is a comment'", - "LOCATION 'file:/tmp'", + "LOCATION 'file:///tmp'", "TBLPROPERTIES (", "'prop1' = '1',", "'prop2' = '2',", From afaabe8db953b49e284182b246d5cd4f61ab03ae Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Mon, 14 Feb 2022 13:53:51 +0800 Subject: [PATCH 3/7] Address comments --- .../org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 1c30f10f63d38..1a3054216972a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -388,7 +388,7 @@ class SessionCatalog( private def makeQualifiedTablePath(locationUri: URI, database: String): URI = { if (locationUri.isAbsolute) { locationUri - } else if (locationUri.getPath.startsWith("/")) { + } else if (new Path(locationUri).isAbsolute) { makeQualifiedPath(locationUri) } else { val dbName = formatDatabaseName(database) From 65b7b184d5755c27e27d5590adc78966fca951ec Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Wed, 16 Feb 2022 12:49:48 +0800 Subject: [PATCH 4/7] Fix tests --- .../spark/sql/execution/command/v1/ShowCreateTableSuite.scala | 2 +- .../spark/sql/execution/command/v2/ShowCreateTableSuite.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala index 1dd5e4a5aaa79..ee8aa424d5c26 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala @@ -53,7 +53,7 @@ trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase |COMMENT 'This is a comment' |TBLPROPERTIES ('prop1' = '1', 'prop2' = '2', 'prop3' = 3, 'prop4' = 4) |PARTITIONED BY (a) - |LOCATION '/tmp' + |LOCATION 'file:/tmp' """.stripMargin) val showDDL = getShowCreateDDL(t) assert(showDDL === Array( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala index 2adbf3cc1c8c0..7c506812079ec 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala @@ -101,7 +101,7 @@ class ShowCreateTableSuite extends command.ShowCreateTableSuiteBase with Command "'via' = '2')", "PARTITIONED BY (a)", "COMMENT 'This is a comment'", - "LOCATION 'file:///tmp'", + "LOCATION 'file:/tmp'", "TBLPROPERTIES (", "'prop1' = '1',", "'prop2' = '2',", From 654b953cfb32c92c8122e7ecec41217d7716c602 Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Fri, 18 Feb 2022 22:54:28 +0800 Subject: [PATCH 5/7] Revert "Fix tests" This reverts commit 65b7b184d5755c27e27d5590adc78966fca951ec. --- .../spark/sql/execution/command/v1/ShowCreateTableSuite.scala | 2 +- .../spark/sql/execution/command/v2/ShowCreateTableSuite.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala index ee8aa424d5c26..1dd5e4a5aaa79 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala @@ -53,7 +53,7 @@ trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase |COMMENT 'This is a comment' |TBLPROPERTIES ('prop1' = '1', 'prop2' = '2', 'prop3' = 3, 'prop4' = 4) |PARTITIONED BY (a) - |LOCATION 'file:/tmp' + |LOCATION '/tmp' """.stripMargin) val showDDL = getShowCreateDDL(t) assert(showDDL === Array( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala index 7c506812079ec..2adbf3cc1c8c0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala @@ -101,7 +101,7 @@ class ShowCreateTableSuite extends command.ShowCreateTableSuiteBase with Command "'via' = '2')", "PARTITIONED BY (a)", "COMMENT 'This is a comment'", - "LOCATION 'file:/tmp'", + "LOCATION 'file:///tmp'", "TBLPROPERTIES (", "'prop1' = '1',", "'prop2' = '2',", From 6782a2356c36cd6b3f6256483e85702f6039a18b Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Fri, 18 Feb 2022 22:54:35 +0800 Subject: [PATCH 6/7] Revert "Split out CommonFileDataSourceSuite" This reverts commit 67c23181d24a0b2e307025122a901af105133d16. --- .../org/apache/spark/sql/avro/AvroSuite.scala | 29 +++-------------- .../source/libsvm/LibSVMRelationSuite.scala | 32 ++++++++----------- .../results/show-create-table.sql.out | 4 +-- .../sql/connector/DataSourceV2SQLSuite.scala | 2 +- .../command/v2/ShowCreateTableSuite.scala | 2 +- 5 files changed, 22 insertions(+), 47 deletions(-) diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index 9172a9aa0db74..d85baeb9386f2 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -53,11 +53,15 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.v2.avro.AvroScan import org.apache.spark.util.Utils -abstract class AvroSuiteBase +abstract class AvroSuite extends QueryTest with SharedSparkSession + with CommonFileDataSourceSuite with NestedDataSourceSuiteBase { + import testImplicits._ + + override protected def dataSourceFormat = "avro" override val nestedDataSources = Seq("avro") val episodesAvro = testFile("episodes.avro") val testAvro = testFile("test.avro") @@ -91,11 +95,6 @@ abstract class AvroSuiteBase } }, new GenericDatumReader[Any]()).getSchema.toString(false) } -} - -abstract class AvroSuite extends AvroSuiteBase { - - import testImplicits._ private def getResourceAvroFilePath(name: String): String = { Thread.currentThread().getContextClassLoader.getResource(name).toString @@ -2435,21 +2434,3 @@ class AvroV2Suite extends AvroSuite with ExplainSuiteHelper { } } } - -class AvroV1SuiteWithCommonFileSourceCheck extends AvroSuiteBase with CommonFileDataSourceSuite { - override protected def sparkConf: SparkConf = - super - .sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "avro") - - override protected def dataSourceFormat = "avro" -} - -class AvroV2SuiteWithCommonFileSourceCheck extends AvroSuiteBase with CommonFileDataSourceSuite { - override protected def sparkConf: SparkConf = - super - .sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "") - - override protected def dataSourceFormat = "avro" -} diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala index d0191ed9b06ca..a456409cfe3bc 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala @@ -32,9 +32,20 @@ import org.apache.spark.sql.execution.datasources.CommonFileDataSourceSuite import org.apache.spark.sql.types.{DoubleType, StructField, StructType} import org.apache.spark.util.Utils -abstract class LibSVMRelationSuiteBase +class LibSVMRelationSuite extends SparkFunSuite - with MLlibTestSparkContext { + with MLlibTestSparkContext + with CommonFileDataSourceSuite { + + override protected def dataSourceFormat = "libsvm" + override protected def inputDataset = { + val rawData = new java.util.ArrayList[Row]() + rawData.add(Row(1.0, Vectors.sparse(1, Seq((0, 1.0))))) + val struct = new StructType() + .add("labelFoo", DoubleType, false) + .add("featuresBar", VectorType, false) + spark.createDataFrame(rawData, struct) + } // Path for dataset var path: String = _ @@ -67,9 +78,6 @@ abstract class LibSVMRelationSuiteBase super.afterAll() } } -} - -class LibSVMRelationSuite extends LibSVMRelationSuiteBase { test("select as sparse vector") { val df = spark.read.format("libsvm").load(path) @@ -218,17 +226,3 @@ class LibSVMRelationSuite extends LibSVMRelationSuiteBase { } } } - -class LibSVMRelationSuiteWithCommonFileSourceCheck - extends LibSVMRelationSuiteBase with CommonFileDataSourceSuite { - - override protected def dataSourceFormat = "libsvm" - override protected def inputDataset = { - val rawData = new java.util.ArrayList[Row]() - rawData.add(Row(1.0, Vectors.sparse(1, Seq((0, 1.0))))) - val struct = new StructType() - .add("labelFoo", DoubleType, false) - .add("featuresBar", VectorType, false) - spark.createDataFrame(rawData, struct) - } -} diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out index ded27abc4c14d..ca1652b337a28 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out @@ -80,7 +80,7 @@ CREATE TABLE default.tbl ( b STRING, c INT) USING parquet -LOCATION 'file:///path/to/table' +LOCATION 'file:/path/to/table' -- !query @@ -110,7 +110,7 @@ CREATE TABLE default.tbl ( b STRING, c INT) USING parquet -LOCATION 'file:///path/to/table' +LOCATION 'file:/path/to/table' -- !query diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index b64ed080d8bf1..d9e3342240bcf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -2773,7 +2773,7 @@ class DataSourceV2SQLSuite val properties = table.properties assert(properties.get(TableCatalog.PROP_PROVIDER) == "parquet") assert(properties.get(TableCatalog.PROP_COMMENT) == "This is a comment") - assert(properties.get(TableCatalog.PROP_LOCATION) == "file:///tmp") + assert(properties.get(TableCatalog.PROP_LOCATION) == "file:/tmp") assert(properties.containsKey(TableCatalog.PROP_OWNER)) assert(properties.get(TableCatalog.PROP_EXTERNAL) == "true") assert(properties.get(s"${TableCatalog.OPTION_PREFIX}from") == "0") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala index 2adbf3cc1c8c0..7c506812079ec 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala @@ -101,7 +101,7 @@ class ShowCreateTableSuite extends command.ShowCreateTableSuiteBase with Command "'via' = '2')", "PARTITIONED BY (a)", "COMMENT 'This is a comment'", - "LOCATION 'file:///tmp'", + "LOCATION 'file:/tmp'", "TBLPROPERTIES (", "'prop1' = '1',", "'prop2' = '2',", From cd16730f0881b4636b6266d3d0d74a224dbcdb47 Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Fri, 18 Feb 2022 23:12:50 +0800 Subject: [PATCH 7/7] Fix tests by initialize SessionCatalog --- .../src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala | 2 ++ .../org/apache/spark/mllib/util/MLlibTestSparkContext.scala | 2 ++ .../resources/sql-tests/results/show-create-table.sql.out | 4 ++-- .../org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala | 2 +- .../spark/sql/execution/command/v1/ShowCreateTableSuite.scala | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index d85baeb9386f2..a6163b01bc69f 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -68,6 +68,8 @@ abstract class AvroSuite override protected def beforeAll(): Unit = { super.beforeAll() + // initialize SessionCatalog here so it has a clean hadoopConf + spark.sessionState.catalog spark.conf.set(SQLConf.FILES_MAX_PARTITION_BYTES.key, 1024) } diff --git a/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala b/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala index 5eb128abacdb9..3a7040d470c05 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala @@ -40,6 +40,8 @@ trait MLlibTestSparkContext extends TempDirectory { self: Suite => .appName("MLlibUnitTest") .getOrCreate() sc = spark.sparkContext + // initialize SessionCatalog here so it has a clean hadoopConf + spark.sessionState.catalog checkpointDir = Utils.createDirectory(tempDir.getCanonicalPath, "checkpoints").toString sc.setCheckpointDir(checkpointDir) diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out index ca1652b337a28..ded27abc4c14d 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out @@ -80,7 +80,7 @@ CREATE TABLE default.tbl ( b STRING, c INT) USING parquet -LOCATION 'file:/path/to/table' +LOCATION 'file:///path/to/table' -- !query @@ -110,7 +110,7 @@ CREATE TABLE default.tbl ( b STRING, c INT) USING parquet -LOCATION 'file:/path/to/table' +LOCATION 'file:///path/to/table' -- !query diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index d9e3342240bcf..b64ed080d8bf1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -2773,7 +2773,7 @@ class DataSourceV2SQLSuite val properties = table.properties assert(properties.get(TableCatalog.PROP_PROVIDER) == "parquet") assert(properties.get(TableCatalog.PROP_COMMENT) == "This is a comment") - assert(properties.get(TableCatalog.PROP_LOCATION) == "file:/tmp") + assert(properties.get(TableCatalog.PROP_LOCATION) == "file:///tmp") assert(properties.containsKey(TableCatalog.PROP_OWNER)) assert(properties.get(TableCatalog.PROP_EXTERNAL) == "true") assert(properties.get(s"${TableCatalog.OPTION_PREFIX}from") == "0") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala index 1dd5e4a5aaa79..ee8aa424d5c26 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala @@ -53,7 +53,7 @@ trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase |COMMENT 'This is a comment' |TBLPROPERTIES ('prop1' = '1', 'prop2' = '2', 'prop3' = 3, 'prop4' = 4) |PARTITIONED BY (a) - |LOCATION '/tmp' + |LOCATION 'file:/tmp' """.stripMargin) val showDDL = getShowCreateDDL(t) assert(showDDL === Array(