From fed1c4e30d37b8a0e9e2f3c87c15bc6cfe0c4534 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 11 Aug 2015 14:22:06 +0900 Subject: [PATCH 1/3] [SPARK-9814][SQL] EqualNotNull not passing to data sources --- .../sql/execution/datasources/DataSourceStrategy.scala | 5 +++++ .../main/scala/org/apache/spark/sql/sources/filters.scala | 8 ++++++++ .../org/apache/spark/sql/sources/FilteredScanSuite.scala | 1 + 3 files changed, 14 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala index 78a4acdf4b1bf..2a4c40db8bb66 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala @@ -349,6 +349,11 @@ private[sql] object DataSourceStrategy extends Strategy with Logging { case expressions.EqualTo(Literal(v, _), a: Attribute) => Some(sources.EqualTo(a.name, v)) + case expressions.EqualNullSafe(a: Attribute, Literal(v, _)) => + Some(sources.EqualNullSafe(a.name, v)) + case expressions.EqualNullSafe(Literal(v, _), a: Attribute) => + Some(sources.EqualNullSafe(a.name, v)) + case expressions.GreaterThan(a: Attribute, Literal(v, _)) => Some(sources.GreaterThan(a.name, v)) case expressions.GreaterThan(Literal(v, _), a: Attribute) => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala index 4d942e4f9287a..de607dcc6582f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala @@ -36,6 +36,14 @@ abstract class Filter */ case class EqualTo(attribute: String, value: Any) extends Filter +/** + * A filter that evaluates to `true` iff the attribute evaluates to a value + * null safe and equal to `value`. + * + * @since 1.5.0 + */ +case class EqualNullSafe(attribute: String, value: Any) extends Filter + /** * A filter that evaluates to `true` iff the attribute evaluates to a value * greater than `value`. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala index 81b3a0f0c5b3a..5ef365797eace 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala @@ -56,6 +56,7 @@ case class SimpleFilteredScan(from: Int, to: Int)(@transient val sqlContext: SQL // Predicate test on integer column def translateFilterOnA(filter: Filter): Int => Boolean = filter match { case EqualTo("a", v) => (a: Int) => a == v + case EqualNullSafe("a", v) => (a: Int) => a == v case LessThan("a", v: Int) => (a: Int) => a < v case LessThanOrEqual("a", v: Int) => (a: Int) => a <= v case GreaterThan("a", v: Int) => (a: Int) => a > v From 7cdcf1c2ff709ecfc4651ff3786bdc1d44a77f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B6=8C=ED=98=81=EC=A7=84?= Date: Tue, 11 Aug 2015 16:41:40 +0900 Subject: [PATCH 2/3] [SPARK-9814][SQL] Fix comments of EqualNullSafe() --- .../main/scala/org/apache/spark/sql/sources/filters.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala index de607dcc6582f..4200f0831ef35 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala @@ -37,8 +37,9 @@ abstract class Filter case class EqualTo(attribute: String, value: Any) extends Filter /** - * A filter that evaluates to `true` iff the attribute evaluates to a value - * null safe and equal to `value`. + * Performs equality comparison, similar to [[EqualTo]]. However, this differs from [[EqualTo]] + * in that it returns `true` (rather than NULL) if both inputs are NULL, and `false` + * (rather than NULL) if one of the input is NULL and the other is not NULL. * * @since 1.5.0 */ From b3fe5d38a7dd808dc79496adca1657307ff86c8e Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 11 Aug 2015 18:19:26 +0900 Subject: [PATCH 3/3] [SPARK-9814][SQL] Remove trailing whitespace --- .../src/main/scala/org/apache/spark/sql/sources/filters.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala index 4200f0831ef35..3780cbbcc9631 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala @@ -37,8 +37,8 @@ abstract class Filter case class EqualTo(attribute: String, value: Any) extends Filter /** - * Performs equality comparison, similar to [[EqualTo]]. However, this differs from [[EqualTo]] - * in that it returns `true` (rather than NULL) if both inputs are NULL, and `false` + * Performs equality comparison, similar to [[EqualTo]]. However, this differs from [[EqualTo]] + * in that it returns `true` (rather than NULL) if both inputs are NULL, and `false` * (rather than NULL) if one of the input is NULL and the other is not NULL. * * @since 1.5.0