diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala index 35cd7c2715869..d70c4b11bc0d7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala @@ -74,10 +74,12 @@ abstract class PartitioningAwareFileIndex( } // retrieve the file metadata filters and reduce to a final filter expression - val fileMetadataFilterOpt = dataFilters.filter(_.references.forall { - case FileSourceMetadataAttribute(_) => true - case _ => false - }).reduceOption(expressions.And) + val fileMetadataFilterOpt = dataFilters.filter { f => + f.references.nonEmpty && f.references.forall { + case FileSourceMetadataAttribute(_) => true + case _ => false + } + }.reduceOption(expressions.And) // - create a bound references for filters: put the metadata struct at 0 position for each file // - retrieve the final metadata struct (could be pruned) from filters diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala index fcaf8df4f9a02..690623e72c994 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala @@ -520,6 +520,18 @@ class FileIndexSuite extends SharedSparkSession { SQLConf.get.setConf(StaticSQLConf.METADATA_CACHE_TTL_SECONDS, previousValue) } } + + test("SPARK-38182: Fix NoSuchElementException if pushed filter does not contain any " + + "references") { + withTable("t") { + withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> + "org.apache.spark.sql.catalyst.optimizer.BooleanSimplification") { + + sql("CREATE TABLE t (c1 int) USING PARQUET") + assert(sql("SELECT * FROM t WHERE c1 = 1 AND 2 > 1").count() == 0) + } + } + } } object DeletionRaceFileSystem {