Skip to content

[SPARK-31818][SQL] Fix pushing down filters with java.time.Instant values in ORC #28636

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 4 commits into from
Closed

[SPARK-31818][SQL] Fix pushing down filters with java.time.Instant values in ORC #28636

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4381ea9
Fix hive-2.3
MaxGekk May 25, 2020
5290d96
Fix hive-1.2
MaxGekk May 25, 2020
e24fb31
timestampToLiteral -> instantToLiteral
MaxGekk May 25, 2020
2773ceb
Rename param: t -> i
MaxGekk May 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.apache.spark.sql.catalyst

import java.sql.{Date, Timestamp}
import java.time.LocalDate
import java.time.{Instant, LocalDate}

import scala.language.implicitConversions

Expand Down Expand Up @@ -152,6 +152,7 @@ package object dsl {
implicit def bigDecimalToLiteral(d: java.math.BigDecimal): Literal = Literal(d)
implicit def decimalToLiteral(d: Decimal): Literal = Literal(d)
implicit def timestampToLiteral(t: Timestamp): Literal = Literal(t)
implicit def timestampToLiteral(t: Instant): Literal = Literal(t)
Copy link
Contributor

@cloud-fan cloud-fan May 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shall we call it instantToLiteral, to be consistent with localDateToLiteral?

implicit def binaryToLiteral(a: Array[Byte]): Literal = Literal(a)

implicit def symbolToUnresolvedAttribute(s: Symbol): analysis.UnresolvedAttribute =
Expand Down
6 changes: 4 additions & 2 deletions sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.spark.sql.execution.datasources.orc

import java.time.LocalDate
import java.time.{Instant, LocalDate}

import org.apache.orc.storage.common.`type`.HiveDecimal
import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
Expand All @@ -26,7 +26,7 @@ import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
import org.apache.orc.storage.serde2.io.HiveDecimalWritable

import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{localDateToDays, toJavaDate}
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types._
Expand Down Expand Up @@ -167,6 +167,8 @@ private[sql] object OrcFilters extends OrcFiltersBase {
new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
case _: DateType if value.isInstanceOf[LocalDate] =>
toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
case _: TimestampType if value.isInstanceOf[Instant] =>
toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
case _ => value
}

Expand Down
56 changes: 34 additions & 22 deletions ...e/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,29 +245,41 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
}

test("filter pushdown - timestamp") {
val timeString = "2015-08-20 14:57:00"
val timestamps = (1 to 4).map { i =>
val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
new Timestamp(milliseconds)
}
withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
val input = Seq(
"1000-01-01 01:02:03",
"1582-10-01 00:11:22",
"1900-01-01 23:59:59",
"2020-05-25 10:11:12").map(Timestamp.valueOf)

checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)

checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)

checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
checkFilterPredicate(Literal(timestamps(0)) <=> $"_1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
withOrcFile(input.map(Tuple1(_))) { path =>
Seq(false, true).foreach { java8Api =>
withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
readFile(path) { implicit df =>
val timestamps = input.map(Literal(_))
checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)

checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)

checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)

checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
checkFilterPredicate(
Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate(
Literal(timestamps(2)) < $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate(
Literal(timestamps(0)) >= $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
}
}
}
}
}

Expand Down
6 changes: 4 additions & 2 deletions sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.spark.sql.execution.datasources.orc

import java.time.LocalDate
import java.time.{Instant, LocalDate}

import org.apache.hadoop.hive.common.`type`.HiveDecimal
import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
Expand All @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable

import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{localDateToDays, toJavaDate}
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types._
Expand Down Expand Up @@ -167,6 +167,8 @@ private[sql] object OrcFilters extends OrcFiltersBase {
new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
case _: DateType if value.isInstanceOf[LocalDate] =>
toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
case _: TimestampType if value.isInstanceOf[Instant] =>
toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
case _ => value
}

Expand Down
54 changes: 33 additions & 21 deletions ...e/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,29 +246,41 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
}

test("filter pushdown - timestamp") {
val timeString = "2015-08-20 14:57:00"
val timestamps = (1 to 4).map { i =>
val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
new Timestamp(milliseconds)
}
withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)

checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
val input = Seq(
"1000-01-01 01:02:03",
"1582-10-01 00:11:22",
"1900-01-01 23:59:59",
"2020-05-25 10:11:12").map(Timestamp.valueOf)

checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
withOrcFile(input.map(Tuple1(_))) { path =>
Seq(false, true).foreach { java8Api =>
withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
readFile(path) { implicit df =>
val timestamps = input.map(Literal(_))
checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)

checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
checkFilterPredicate(
Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)

checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)

checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
checkFilterPredicate(
Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
checkFilterPredicate(
Literal(timestamps(2)) < $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate(
Literal(timestamps(0)) >= $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
}
}
}
}
}

Expand Down