-
Notifications
You must be signed in to change notification settings - Fork 29.3k
[SPARK-30184][SQL] Implement a helper method for aliasing functions #26808
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 10 commits
29fdaba
7d5f4be
7ba7802
759262d
5ec101f
f210bb9
cc234b9
7617ec3
92e381b
a2d75de
3ef62af
87c6ea3
9480532
5c540fc
d780dfc
85d9597
a71e8a7
dd2d85d
c1b3afb
e7a4e90
ca886f0
125cfac
4ca20f4
aecdd8a
bbd4397
9146913
8e9e42b
36418e2
ce8ea17
4b536dd
737f33a
1920940
700a84d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,7 +24,8 @@ import org.apache.spark.sql.types.IntegerType | |
| class LastTestSuite extends SparkFunSuite { | ||
| val input = AttributeReference("input", IntegerType, nullable = true)() | ||
| val evaluator = DeclarativeAggregateEvaluator(Last(input, Literal(false)), Seq(input)) | ||
| val evaluatorIgnoreNulls = DeclarativeAggregateEvaluator(Last(input, Literal(true)), Seq(input)) | ||
| val evaluatorIgnoreNulls = DeclarativeAggregateEvaluator( | ||
| Last(input, Literal(true)), Seq(input)) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: unnecessary change |
||
|
|
||
| test("empty buffer") { | ||
| assert(evaluator.initialize() === InternalRow(null, false)) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -794,12 +794,18 @@ class ExpressionParserSuite extends AnalysisTest { | |
| } | ||
|
|
||
| test("Support respect nulls keywords for first_value and last_value") { | ||
| assertEqual("first_value(a ignore nulls)", First('a, Literal(true)).toAggregateExpression()) | ||
| assertEqual("first_value(a respect nulls)", First('a, Literal(false)).toAggregateExpression()) | ||
| assertEqual("first_value(a)", First('a, Literal(false)).toAggregateExpression()) | ||
| assertEqual("last_value(a ignore nulls)", Last('a, Literal(true)).toAggregateExpression()) | ||
| assertEqual("last_value(a respect nulls)", Last('a, Literal(false)).toAggregateExpression()) | ||
| assertEqual("last_value(a)", Last('a, Literal(false)).toAggregateExpression()) | ||
| assertEqual("first_value(a ignore nulls)", | ||
| First('a, Literal(true)).toAggregateExpression()) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ditto, unnecessary change |
||
| assertEqual("first_value(a respect nulls)", | ||
| First('a, Literal(false)).toAggregateExpression()) | ||
| assertEqual("first_value(a)", | ||
| First('a, Literal(false)).toAggregateExpression()) | ||
| assertEqual("last_value(a ignore nulls)", | ||
| Last('a, Literal(true)).toAggregateExpression()) | ||
| assertEqual("last_value(a respect nulls)", | ||
| Last('a, Literal(false)).toAggregateExpression()) | ||
| assertEqual("last_value(a)", | ||
| Last('a, Literal(false)).toAggregateExpression()) | ||
| } | ||
|
|
||
| test("timestamp literals") { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -254,7 +254,8 @@ object StatFunctions extends Logging { | |
| stats.toLowerCase(Locale.ROOT) match { | ||
| case "count" => (child: Expression) => Count(child).toAggregateExpression() | ||
| case "mean" => (child: Expression) => Average(child).toAggregateExpression() | ||
| case "stddev" => (child: Expression) => StddevSamp(child).toAggregateExpression() | ||
| case "stddev" => (child: Expression) => | ||
| StddevSamp("stddev_samp", child).toAggregateExpression() | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. shall we use |
||
| case "min" => (child: Expression) => Min(child).toAggregateExpression() | ||
| case "max" => (child: Expression) => Max(child).toAggregateExpression() | ||
| case _ => throw new IllegalArgumentException(s"$stats is not a recognised statistic") | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -85,7 +85,7 @@ struct<stddev_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double> | |
| -- !query 10 | ||
| SELECT udf(stddev_samp(b)) FROM aggtest | ||
| -- !query 10 schema | ||
| struct<CAST(udf(cast(stddev_samp(cast(b as double)) as string)) AS DOUBLE):double> | ||
| struct<CAST(udf(cast(stddev_samp(stddev_samp, cast(b as double)) as string)) AS DOUBLE):double> | ||
| -- !query 10 output | ||
| 151.38936080399804 | ||
|
|
||
|
|
@@ -101,7 +101,7 @@ struct<var_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double> | |
| -- !query 12 | ||
| SELECT udf(var_samp(b)) FROM aggtest | ||
| -- !query 12 schema | ||
| struct<CAST(udf(cast(var_samp(cast(b as double)) as string)) AS DOUBLE):double> | ||
| struct<CAST(udf(cast(var_samp(var_samp, cast(b as double)) as string)) AS DOUBLE):double> | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This looks weird. Looking at
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It seems |
||
| -- !query 12 output | ||
| 22918.738564643096 | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.