From f4b85576cb381fd5cd68c854af6a855779d939e8 Mon Sep 17 00:00:00 2001 From: BartekH Date: Mon, 15 May 2017 14:08:41 +0200 Subject: [PATCH 1/5] Add "full_outer" name to join types I have discovered that "full_outer" name option is working in Spark 2.0, but it is not printed in exception. Please verify. --- .../scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala index 90d11d6d91512..8c55a884dbec6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute object JoinType { def apply(typ: String): JoinType = typ.toLowerCase(Locale.ROOT).replace("_", "") match { case "inner" => Inner - case "outer" | "full" | "fullouter" => FullOuter + case "outer" | "full" | "fullouter" | "full_outer" => FullOuter case "leftouter" | "left" => LeftOuter case "rightouter" | "right" => RightOuter case "leftsemi" => LeftSemi @@ -33,7 +33,7 @@ object JoinType { case _ => val supported = Seq( "inner", - "outer", "full", "fullouter", + "outer", "full", "fullouter", "full_outer", "leftouter", "left", "rightouter", "right", "leftsemi", From 93d905cc896cd0ded040010c3d9dfd6ce4b2de0d Mon Sep 17 00:00:00 2001 From: BartekH Date: Tue, 6 Jun 2017 15:52:54 +0200 Subject: [PATCH 2/5] Update joinTypes.scala --- .../apache/spark/sql/catalyst/plans/joinTypes.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala index 8c55a884dbec6..c77849035a975 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute object JoinType { def apply(typ: String): JoinType = typ.toLowerCase(Locale.ROOT).replace("_", "") match { case "inner" => Inner - case "outer" | "full" | "fullouter" | "full_outer" => FullOuter + case "outer" | "full" | "fullouter" => FullOuter case "leftouter" | "left" => LeftOuter case "rightouter" | "right" => RightOuter case "leftsemi" => LeftSemi @@ -34,10 +34,10 @@ object JoinType { val supported = Seq( "inner", "outer", "full", "fullouter", "full_outer", - "leftouter", "left", - "rightouter", "right", - "leftsemi", - "leftanti", + "leftouter", "left", "left_outer", + "rightouter", "right", "right_outer", + "leftsemi", "left_semi", + "leftanti", "left_anti", "cross") throw new IllegalArgumentException(s"Unsupported join type '$typ'. " + From 71d65e84b326ad94a98e46dfedee172a14525cbd Mon Sep 17 00:00:00 2001 From: BartekH Date: Tue, 6 Jun 2017 16:55:00 +0200 Subject: [PATCH 3/5] Create JoinTypesTest.scala tests for JoinType object --- .../sql/catalyst/plans/JoinTypesTest.scala | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala new file mode 100644 index 0000000000000..c62095ef725cc --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala @@ -0,0 +1,44 @@ +package org.apache.spark.sql.catalyst.plans + +import org.apache.spark.SparkFunSuite + +class JoinTypesTest extends SparkFunSuite { + + test("construct an Inner type") { + assert(JoinType.apply("inner") === Inner) + } + + test("construct a FullOuter type") { + assert(JoinType.apply("fullouter") === FullOuter) + assert(JoinType.apply("full_outer") === FullOuter) + assert(JoinType.apply("outer") === FullOuter) + assert(JoinType.apply("full") === FullOuter) + } + + test("construct a LeftOuter type") { + assert(JoinType.apply("leftouter") === LeftOuter) + assert(JoinType.apply("left_outer") === LeftOuter) + assert(JoinType.apply("left") === LeftOuter) + } + + test("construct a RightOuter type") { + assert(JoinType.apply("rightouter") === RightOuter) + assert(JoinType.apply("right_outer") === RightOuter) + assert(JoinType.apply("right") === RightOuter) + } + + test("construct a LeftSemi type") { + assert(JoinType.apply("leftsemi") === LeftSemi) + assert(JoinType.apply("left_semi") === LeftSemi) + } + + test("construct a LeftAnti type") { + assert(JoinType.apply("leftanti") === LeftAnti) + assert(JoinType.apply("left_anti") === LeftAnti) + } + + test("construct a Cross type") { + assert(JoinType.apply("cross") === Cross) + } + +} From 7bea0ad405943644a68102c91011cf00a4f305ac Mon Sep 17 00:00:00 2001 From: BartekH Date: Tue, 6 Jun 2017 19:33:35 +0200 Subject: [PATCH 4/5] Update JoinTypesTest.scala add Apache license on the top of file --- .../sql/catalyst/plans/JoinTypesTest.scala | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala index c62095ef725cc..d9474fc9400b4 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.spark.sql.catalyst.plans import org.apache.spark.SparkFunSuite From 9fc9a0ad567dfb28d22d94321fcef0ea3b1ae73b Mon Sep 17 00:00:00 2001 From: BartekH Date: Wed, 28 Jun 2017 11:44:41 +0200 Subject: [PATCH 5/5] remove apply method from JoinType execution --- .../sql/catalyst/plans/JoinTypesTest.scala | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala index d9474fc9400b4..d56f97970122c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala @@ -23,40 +23,40 @@ import org.apache.spark.SparkFunSuite class JoinTypesTest extends SparkFunSuite { test("construct an Inner type") { - assert(JoinType.apply("inner") === Inner) + assert(JoinType("inner") === Inner) } test("construct a FullOuter type") { - assert(JoinType.apply("fullouter") === FullOuter) - assert(JoinType.apply("full_outer") === FullOuter) - assert(JoinType.apply("outer") === FullOuter) - assert(JoinType.apply("full") === FullOuter) + assert(JoinType("fullouter") === FullOuter) + assert(JoinType("full_outer") === FullOuter) + assert(JoinType("outer") === FullOuter) + assert(JoinType("full") === FullOuter) } test("construct a LeftOuter type") { - assert(JoinType.apply("leftouter") === LeftOuter) - assert(JoinType.apply("left_outer") === LeftOuter) - assert(JoinType.apply("left") === LeftOuter) + assert(JoinType("leftouter") === LeftOuter) + assert(JoinType("left_outer") === LeftOuter) + assert(JoinType("left") === LeftOuter) } test("construct a RightOuter type") { - assert(JoinType.apply("rightouter") === RightOuter) - assert(JoinType.apply("right_outer") === RightOuter) - assert(JoinType.apply("right") === RightOuter) + assert(JoinType("rightouter") === RightOuter) + assert(JoinType("right_outer") === RightOuter) + assert(JoinType("right") === RightOuter) } test("construct a LeftSemi type") { - assert(JoinType.apply("leftsemi") === LeftSemi) - assert(JoinType.apply("left_semi") === LeftSemi) + assert(JoinType("leftsemi") === LeftSemi) + assert(JoinType("left_semi") === LeftSemi) } test("construct a LeftAnti type") { - assert(JoinType.apply("leftanti") === LeftAnti) - assert(JoinType.apply("left_anti") === LeftAnti) + assert(JoinType("leftanti") === LeftAnti) + assert(JoinType("left_anti") === LeftAnti) } test("construct a Cross type") { - assert(JoinType.apply("cross") === Cross) + assert(JoinType("cross") === Cross) } }