[SPARK-2700] [SQL] Hidden files (such as .impala_insert_staging) should be filtered out by sqlContext.parquetFile

chutium · marmbrus · commit b7c89a7f0ca7 · 2014-08-08T13:31:08.000-07:00
Author: chutium <teng.qiu@gmail.com> Closes #1691 from chutium/SPARK-2700 and squashes the following commits: b76ae8c [chutium] [SPARK-2700] [SQL] fixed styling issue d75a8bd [chutium] [SPARK-2700] [SQL] Hidden files (such as .impala_insert_staging) should be filtered out by sqlContext.parquetFile
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala
@@ -373,8 +373,9 @@ private[parquet] object ParquetTypesConverter extends Logging {
     }
     ParquetRelation.enableLogForwarding()
 
-    val children = fs.listStatus(path).filterNot {
-      _.getPath.getName == FileOutputCommitter.SUCCEEDED_FILE_NAME
+    val children = fs.listStatus(path).filterNot { status =>
+      val name = status.getPath.getName
+      name(0) == '.' || name == FileOutputCommitter.SUCCEEDED_FILE_NAME
     }
 
     // NOTE (lian): Parquet "_metadata" file can be very slow if the file consists of lots of row

Original file line number	Diff line number	Diff line change
`@@ -373,8 +373,9 @@ private[parquet] object ParquetTypesConverter extends Logging {`
`373`	`373`	`}`
`374`	`374`	`ParquetRelation.enableLogForwarding()`
`375`	`375`
`376`		`- val children = fs.listStatus(path).filterNot {`
`377`		`- _.getPath.getName == FileOutputCommitter.SUCCEEDED_FILE_NAME`
	`376`	`+ val children = fs.listStatus(path).filterNot { status =>`
	`377`	`+ val name = status.getPath.getName`
	`378`	`+ name(0) == '.' \|\| name == FileOutputCommitter.SUCCEEDED_FILE_NAME`
`378`	`379`	`}`
`379`	`380`
`380`	`381`	`// NOTE (lian): Parquet "_metadata" file can be very slow if the file consists of lots of row`