Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ case class InsertIntoHadoopFsRelationCommand(
sparkSession.catalog.refreshByPath(outputPath.toString)

if (catalogTable.nonEmpty) {
sparkSession.catalog.refreshTable(catalogTable.get.identifier.quotedString)
Comment thread
wangyum marked this conversation as resolved.
Outdated
CommandUtils.updateTableStats(sparkSession, catalogTable.get)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we switch line 193 and 192?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is a issue here: It may refresh the table twice. I moved this logical to CommandUtils.updateTableStats:

// In other cases, we still need to invalidate the table relation cache.
catalog.refreshTable(table.identifier)

}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,7 @@ class HiveParquetMetastoreSuite extends ParquetPartitioningTest {
def checkCached(tableIdentifier: TableIdentifier): Unit = {
// Converted test_parquet should be cached.
getCachedDataSourceTable(tableIdentifier) match {
case null => fail("Converted test_parquet should be cached in the cache.")
case null => fail(s"Converted ${tableIdentifier.table} should be cached in the cache.")
case LogicalRelation(_: HadoopFsRelation, _, _, _) => // OK
case other =>
fail(
Expand Down Expand Up @@ -480,7 +480,7 @@ class HiveParquetMetastoreSuite extends ParquetPartitioningTest {
|INSERT INTO TABLE test_insert_parquet
|select a, b from jt
""".stripMargin)
checkCached(tableIdentifier)
assert(getCachedDataSourceTable(tableIdentifier) === null)
// Make sure we can read the data.
checkAnswer(
sql("select * from test_insert_parquet"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ import org.scalatest.BeforeAndAfterEach

import org.apache.spark.SparkException
import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, TableAlreadyExistsException}
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.execution.command.{DDLSuite, DDLUtils}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.HiveExternalCatalog
import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
import org.apache.spark.sql.hive.HiveUtils.{CONVERT_METASTORE_ORC, CONVERT_METASTORE_PARQUET}
import org.apache.spark.sql.hive.orc.OrcFileOperator
import org.apache.spark.sql.hive.test.TestHiveSingleton
Expand Down Expand Up @@ -2370,4 +2370,26 @@ class HiveDDLSuite
))
}
}

test("Refresh table after insert into table") {
withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") {
Seq("t1", "t2").foreach { tableName =>
withTable(tableName) {
if (tableName.equals("t1")) {
sql(s"CREATE TABLE $tableName (a INT) STORED AS parquet")
} else {
sql(s"CREATE TABLE $tableName (a INT) USING parquet")
}

sql(s"INSERT INTO TABLE $tableName VALUES (1)")

val catalog = spark.sessionState.catalog
val qualifiedTableName = QualifiedTableName(catalog.getCurrentDatabase, tableName)
val cachedRelation = catalog.getCachedTable(qualifiedTableName)
// cachedRelation should be null after refresh table.
assert(cachedRelation === null)
Comment thread
wangyum marked this conversation as resolved.
Outdated
}
}
}
}
}