Merge pull request #199 from ianlcsd/SPY-1453

SPY-1453: disable LogicalRelation for CSD and include identifier in simple String

Author: ianlcsd

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala

@@ -16,6 +16,7 @@
  */
 package org.apache.spark.sql.execution.datasources
 
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.catalyst.expressions.{AttributeMap, AttributeReference}
@@ -69,7 +70,10 @@ case class LogicalRelation(
     case _ =>  // Do nothing.
   }
 
-  override def simpleString: String = s"Relation[${Utils.truncatedString(output, ",")}] $relation"
+  override def simpleString: String = {
+    s"Relation[${Utils.truncatedString(output, ",")}] $relation " +
+      s"${catalogTable.map(_.identifier.identifier).getOrElse("")}"
+  }
 }
 
 object LogicalRelation {

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala

@@ -190,33 +190,20 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log
         } else {
           selectParquetLocationDirectories(relation.tableMeta.identifier.table, Option(rootPath))
         }
-      withTableCreationLock(tableIdentifier, {
-        val cached = getCached(
-          tableIdentifier,
-          paths,
-          metastoreSchema,
-          fileFormatClass,
-          None)
-        val logicalRelation = cached.getOrElse {
-          val (dataSchema, updatedTable) = inferIfNeeded(relation, options, fileFormat)
-          val created =
-            LogicalRelation(
-              DataSource(
-                sparkSession = sparkSession,
-                paths = paths.map(_.toString),
-                userSpecifiedSchema = Option(dataSchema),
-                // We don't support hive bucketed tables, only ones we write out.
-                bucketSpec = None,
-                options = options,
-                className = fileType).resolveRelation(),
-              table = updatedTable)
-
-          catalogProxy.cacheTable(tableIdentifier, created)
-          created
-        }
-
-        logicalRelation
-      })
+      // SPY-1453: disable the cache completely until we migrated to partitioning
+      val (dataSchema, updatedTable) = inferIfNeeded(relation, options, fileFormat)
+      val created =
+        LogicalRelation(
+          DataSource(
+            sparkSession = sparkSession,
+            paths = paths.map(_.toString),
+            userSpecifiedSchema = Option(dataSchema),
+            // We don't support hive bucketed tables, only ones we write out.
+            bucketSpec = None,
+            options = options,
+            className = fileType).resolveRelation(),
+          table = updatedTable)
+      created
     }
     // The inferred schema may have different field names as the table schema, we should respect
     // it, but also respect the exprId in table relation output.