From 673ef10e19ceef6b03129e289c4fab45d3585f92 Mon Sep 17 00:00:00 2001 From: Cheng Hao Date: Thu, 18 Jun 2015 19:15:48 +0800 Subject: [PATCH 1/4] Give default field names for CreateStruct --- .../spark/sql/catalyst/expressions/complexTypes.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala index 72fdcebb4cbc..4c4264e90983 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala @@ -63,12 +63,7 @@ case class CreateStruct(children: Seq[Expression]) extends Expression { assert(resolved, s"CreateStruct contains unresolvable children: ${children.filterNot(_.resolved)}.") val fields = children.zipWithIndex.map { case (child, idx) => - child match { - case ne: NamedExpression => - StructField(ne.name, ne.dataType, ne.nullable, ne.metadata) - case _ => - StructField(s"col${idx + 1}", child.dataType, child.nullable, Metadata.empty) - } + StructField(s"col${idx + 1}", child.dataType, child.nullable, Metadata.empty) } StructType(fields) } From b49227671574db6284ddaad016e5d30b96788f2a Mon Sep 17 00:00:00 2001 From: Cheng Hao Date: Thu, 18 Jun 2015 19:22:13 +0800 Subject: [PATCH 2/4] scalastyle --- .../spark/sql/catalyst/expressions/complexTypes.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala index 4c4264e90983..09e2f01fb24b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala @@ -62,9 +62,11 @@ case class CreateStruct(children: Seq[Expression]) extends Expression { override lazy val dataType: StructType = { assert(resolved, s"CreateStruct contains unresolvable children: ${children.filterNot(_.resolved)}.") - val fields = children.zipWithIndex.map { case (child, idx) => - StructField(s"col${idx + 1}", child.dataType, child.nullable, Metadata.empty) - } + + // Like the GenericUDFStruct of Hive, we will give the default names for each of its fields. + val fields = children.zipWithIndex.map { case (child, idx) => + StructField(s"col${idx + 1}", child.dataType, child.nullable, Metadata.empty) + } StructType(fields) } From 35fa5fbd7b97879acdf1d2027ed0fa587b8ae301 Mon Sep 17 00:00:00 2001 From: Cheng Hao Date: Thu, 18 Jun 2015 04:30:08 -0700 Subject: [PATCH 3/4] fix the bugs in unittest for CreateStruct --- .../org/apache/spark/sql/DataFrameFunctionsSuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala index cfd23867a9bb..19061d25b1f6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala @@ -60,8 +60,8 @@ class DataFrameFunctionsSuite extends QueryTest { val row = df.select(struct("a", "b")).first() val expectedType = StructType(Seq( - StructField("a", IntegerType, nullable = false), - StructField("b", StringType) + StructField("col1", IntegerType, nullable = false), + StructField("col2", StringType) )) assert(row.schema(0).dataType === expectedType) assert(row.getAs[Row](0) === Row(1, "str")) @@ -72,8 +72,8 @@ class DataFrameFunctionsSuite extends QueryTest { val row = df.select(struct((col("a") * 2).as("c"), col("b"))).first() val expectedType = StructType(Seq( - StructField("c", IntegerType, nullable = false), - StructField("b", StringType) + StructField("col1", IntegerType, nullable = false), + StructField("col2", StringType) )) assert(row.schema(0).dataType === expectedType) assert(row.getAs[Row](0) === Row(2, "str")) From 2efe8ba0ad1a8371c0493b7e247a683156da17b0 Mon Sep 17 00:00:00 2001 From: Cheng Hao Date: Thu, 18 Jun 2015 05:44:32 -0700 Subject: [PATCH 4/4] remove the unused code/comment/unittest --- .../src/main/scala/org/apache/spark/sql/functions.scala | 6 ++---- .../org/apache/spark/sql/DataFrameFunctionsSuite.scala | 6 ------ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index dff0932c450a..a0a38e17ec67 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -714,16 +714,14 @@ object functions { /** * Creates a new struct column. The input column must be a column in a [[DataFrame]], or - * a derived column expression that is named (i.e. aliased). + * a derived column expression. * * @group normal_funcs * @since 1.4.0 */ @scala.annotation.varargs def struct(cols: Column*): Column = { - require(cols.forall(_.expr.isInstanceOf[NamedExpression]), - s"struct input columns must all be named or aliased ($cols)") - CreateStruct(cols.map(_.expr.asInstanceOf[NamedExpression])) + CreateStruct(cols.map(_.expr)) } /** diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala index 19061d25b1f6..aad07dfe91e7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala @@ -79,12 +79,6 @@ class DataFrameFunctionsSuite extends QueryTest { assert(row.getAs[Row](0) === Row(2, "str")) } - test("struct: must use named column expression") { - intercept[IllegalArgumentException] { - struct(col("a") * 2) - } - } - test("constant functions") { checkAnswer( testData2.select(e()).limit(1),