Pull commits in to release 2.5.0-palantir.X (apache#539)

* [SPARK-27267][CORE] Update snappy to avoid error when decompressing empty serialized data (apache#531)
* [SPARK-27514][SQL] Skip collapsing windows with empty window expressions (apache#538)
* Bump hadoop to 2.9.2-palantir.5 (apache#537)

Author: vinooganesh

dev/deps/spark-deps-hadoop-palantir

@@ -63,26 +63,26 @@ gson-2.2.4.jar
 guava-14.0.1.jar
 guice-3.0.jar
 guice-servlet-3.0.jar
-hadoop-annotations-2.9.2-palantir.4.jar
-hadoop-auth-2.9.2-palantir.4.jar
-hadoop-aws-2.9.2-palantir.4.jar
-hadoop-azure-2.9.2-palantir.4.jar
-hadoop-azure-datalake-2.9.2-palantir.4.jar
-hadoop-client-2.9.2-palantir.4.jar
-hadoop-common-2.9.2-palantir.4.jar
-hadoop-hdfs-client-2.9.2-palantir.4.jar
-hadoop-mapreduce-client-app-2.9.2-palantir.4.jar
-hadoop-mapreduce-client-common-2.9.2-palantir.4.jar
-hadoop-mapreduce-client-core-2.9.2-palantir.4.jar
-hadoop-mapreduce-client-jobclient-2.9.2-palantir.4.jar
-hadoop-mapreduce-client-shuffle-2.9.2-palantir.4.jar
-hadoop-openstack-2.9.2-palantir.4.jar
-hadoop-yarn-api-2.9.2-palantir.4.jar
-hadoop-yarn-client-2.9.2-palantir.4.jar
-hadoop-yarn-common-2.9.2-palantir.4.jar
-hadoop-yarn-registry-2.9.2-palantir.4.jar
-hadoop-yarn-server-common-2.9.2-palantir.4.jar
-hadoop-yarn-server-web-proxy-2.9.2-palantir.4.jar
+hadoop-annotations-2.9.2-palantir.5.jar
+hadoop-auth-2.9.2-palantir.5.jar
+hadoop-aws-2.9.2-palantir.5.jar
+hadoop-azure-2.9.2-palantir.5.jar
+hadoop-azure-datalake-2.9.2-palantir.5.jar
+hadoop-client-2.9.2-palantir.5.jar
+hadoop-common-2.9.2-palantir.5.jar
+hadoop-hdfs-client-2.9.2-palantir.5.jar
+hadoop-mapreduce-client-app-2.9.2-palantir.5.jar
+hadoop-mapreduce-client-common-2.9.2-palantir.5.jar
+hadoop-mapreduce-client-core-2.9.2-palantir.5.jar
+hadoop-mapreduce-client-jobclient-2.9.2-palantir.5.jar
+hadoop-mapreduce-client-shuffle-2.9.2-palantir.5.jar
+hadoop-openstack-2.9.2-palantir.5.jar
+hadoop-yarn-api-2.9.2-palantir.5.jar
+hadoop-yarn-client-2.9.2-palantir.5.jar
+hadoop-yarn-common-2.9.2-palantir.5.jar
+hadoop-yarn-registry-2.9.2-palantir.5.jar
+hadoop-yarn-server-common-2.9.2-palantir.5.jar
+hadoop-yarn-server-web-proxy-2.9.2-palantir.5.jar
 hibernate-validator-5.2.4.Final.jar
 hk2-api-2.5.0-b32.jar
 hk2-locator-2.5.0-b32.jar
@@ -192,7 +192,7 @@ shapeless_2.11-2.3.2.jar
 slf4j-api-1.7.25.jar
 slf4j-log4j12-1.7.25.jar
 snakeyaml-1.23.jar
-snappy-java-1.1.7.2.jar
+snappy-java-1.1.7.3.jar
 spire-macros_2.11-0.13.0.jar
 spire_2.11-0.13.0.jar
 stax-api-1.0-2.jar

dists/hadoop-palantir-bom/pom.xml

@@ -34,7 +34,7 @@
         <sbt.project.name>spark-dist-hadoop-palantir-bom</sbt.project.name>
         <build.testJarPhase>none</build.testJarPhase>
         <build.copyDependenciesPhase>none</build.copyDependenciesPhase>
-        <hadoop.version>2.9.2-palantir.4</hadoop.version>
+        <hadoop.version>2.9.2-palantir.5</hadoop.version>
         <curator.version>2.7.1</curator.version>
     </properties>
 

pom.xml

@@ -127,7 +127,7 @@
     <sbt.project.name>spark</sbt.project.name>
     <slf4j.version>1.7.25</slf4j.version>
     <log4j.version>1.2.17</log4j.version>
-    <hadoop.version>2.9.2-palantir.4</hadoop.version>
+    <hadoop.version>2.9.2-palantir.5</hadoop.version>
     <protobuf.version>2.5.0</protobuf.version>
     <yarn.version>${hadoop.version}</yarn.version>
     <zookeeper.version>3.4.7</zookeeper.version>
@@ -180,7 +180,7 @@
     <scalafmt.skip>true</scalafmt.skip>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
     <fasterxml.jackson.version>2.9.7</fasterxml.jackson.version>
-    <snappy.version>1.1.7.2</snappy.version>
+    <snappy.version>1.1.7.3</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
     <calcite.version>1.2.0-incubating</calcite.version>
     <commons-codec.version>1.11</commons-codec.version>

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

@@ -798,6 +798,7 @@ object CollapseWindow extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
     case w1 @ Window(we1, ps1, os1, w2 @ Window(we2, ps2, os2, grandChild))
         if ps1 == ps2 && os1 == os2 && w1.references.intersect(w2.windowOutputSet).isEmpty &&
+          we1.nonEmpty && we2.nonEmpty &&
           // This assumes Window contains the same type of window expressions. This is ensured
           // by ExtractWindowFunctions.
           WindowFunctionType.functionType(we1.head) == WindowFunctionType.functionType(we2.head) =>

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala

@@ -89,4 +89,15 @@ class CollapseWindowSuite extends PlanTest {
     val optimized = Optimize.execute(query.analyze)
     comparePlans(optimized, expected)
   }
+
+  test("Skip windows with empty window expressions") {
+    val query = testRelation
+      .window(Seq(), partitionSpec1, orderSpec1)
+      .window(Seq(sum(a).as('sum_a)), partitionSpec1, orderSpec1)
+
+    val optimized = Optimize.execute(query.analyze)
+    val correctAnswer = query.analyze
+
+    comparePlans(optimized, correctAnswer)
+  }
 }