Rerun plan stability suite

Author: tanelk

.github/workflows/build_and_test.yml

@@ -126,7 +126,7 @@ jobs:
     - name: Install Python packages (Python 3.8)
       if: (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
       run: |
-        python3.8 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
+        python3.8 -m pip install numpy 'pyarrow<5.0.0' pandas scipy xmlrunner
         python3.8 -m pip list
     # Run the tests.
     - name: Run tests
@@ -217,7 +217,7 @@ jobs:
     # Ubuntu 20.04. See also SPARK-33162.
     - name: Install Python packages (Python 3.6)
       run: |
-        python3.6 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner 'plotly>=4.8'
+        python3.6 -m pip install numpy 'pyarrow<4.0.0' pandas scipy xmlrunner 'plotly>=4.8'
         python3.6 -m pip list
     - name: List Python packages (Python 3.9)
       run: |
@@ -388,7 +388,7 @@ jobs:
         # Jinja2 3.0.0+ causes error when building with Sphinx.
         #   See also https://issues.apache.org/jira/browse/SPARK-35375.
         python3.6 -m pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme ipython nbsphinx numpydoc 'jinja2<3.0.0'
-        python3.6 -m pip install sphinx_plotly_directive 'pyarrow<3.0.0' pandas 'plotly>=4.8'
+        python3.6 -m pip install sphinx_plotly_directive 'pyarrow<5.0.0' pandas 'plotly>=4.8'
         apt-get update -y
         apt-get install -y ruby ruby-dev
         Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
@@ -455,7 +455,7 @@ jobs:
         java-version: 11
     - name: Build with Maven
       run: |
-        export MAVEN_OPTS="-Xss16m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
         export MAVEN_CLI_OPTS="--no-transfer-progress"
         # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414.
         ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 install

.gitignore

@@ -15,10 +15,10 @@
 .ensime_cache/
 .ensime_lucene
 .generated-mima*
-# The star is required for further !.idea/ to work, see https://git-scm.com/docs/gitignore
-.idea/*
+# All the files under .idea/ are ignore. To add new files under ./idea that are not in the VCS yet, please use `git add -f`
+.idea/
+# SPARK-35223: Add IssueNavigationLink to make IDEA support hyperlink on JIRA Ticket and GitHub PR on Git plugin.
 !.idea/vcs.xml
-python/.idea
 .idea_modules/
 .project
 .pydevproject

README.md

@@ -9,6 +9,7 @@ and Structured Streaming for stream processing.
 
 <https://spark.apache.org/>
 
+[![GitHub Action Build](https://github.com/apache/spark/actions/workflows/build_and_test.yml/badge.svg?branch=master)](https://github.com/apache/spark/actions/workflows/build_and_test.yml?query=branch%3Amaster)
 [![Jenkins Build](https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-3.2/badge/icon)](https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-3.2)
 [![AppVeyor Build](https://img.shields.io/appveyor/ci/ApacheSoftwareFoundation/spark/master.svg?style=plastic&logo=appveyor)](https://ci.appveyor.com/project/ApacheSoftwareFoundation/spark)
 [![PySpark Coverage](https://img.shields.io/badge/dynamic/xml.svg?label=pyspark%20coverage&url=https%3A%2F%2Fspark-test.github.io%2Fpyspark-coverage-site&query=%2Fhtml%2Fbody%2Fdiv%5B1%5D%2Fdiv%2Fh1%2Fspan&colorB=brightgreen&style=plastic)](https://spark-test.github.io/pyspark-coverage-site)

build/sbt

@@ -53,7 +53,7 @@ realpath () {
 declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
 declare -r sbt_opts_file=".sbtopts"
 declare -r etc_sbt_opts_file="/etc/sbt/sbtopts"
-declare -r default_sbt_opts="-Xss16m"
+declare -r default_sbt_opts="-Xss64m"
 
 usage() {
  cat <<EOM

common/kvstore/src/test/java/org/apache/spark/util/kvstore/DBIteratorSuite.java

@@ -380,7 +380,7 @@ public void childIndexDescendingWithLast() throws Exception {
 
   @Test
   public void testRefWithIntNaturalKey() throws Exception {
-    LevelDBSuite.IntKeyType i = new LevelDBSuite.IntKeyType();
+    IntKeyType i = new IntKeyType();
     i.key = 1;
     i.id = "1";
     i.values = Arrays.asList("1");

common/kvstore/src/test/java/org/apache/spark/util/kvstore/IntKeyType.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util.kvstore;
+
+import java.util.List;
+
+public class IntKeyType {
+
+  @KVIndex
+  public int key;
+
+  @KVIndex("id")
+  public String id;
+
+  public List<String> values;
+
+  @Override
+  public boolean equals(Object o) {
+    if (o instanceof IntKeyType) {
+      IntKeyType other = (IntKeyType) o;
+      return key == other.key && id.equals(other.id) && values.equals(other.values);
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return id.hashCode();
+  }
+
+}
+

common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java

@@ -338,31 +338,4 @@ private int countKeys(Class<?> type) throws Exception {
 
     return count;
   }
-
-  public static class IntKeyType {
-
-    @KVIndex
-    public int key;
-
-    @KVIndex("id")
-    public String id;
-
-    public List<String> values;
-
-    @Override
-    public boolean equals(Object o) {
-      if (o instanceof IntKeyType) {
-        IntKeyType other = (IntKeyType) o;
-        return key == other.key && id.equals(other.id) && values.equals(other.values);
-      }
-      return false;
-    }
-
-    @Override
-    public int hashCode() {
-      return id.hashCode();
-    }
-
-  }
-
 }

common/network-common/src/main/java/org/apache/spark/network/client/BaseResponseCallback.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.client;
+
+/**
+ * A basic callback. This is extended by {@link RpcResponseCallback} and
+ * {@link MergedBlockMetaResponseCallback} so that both RpcRequests and MergedBlockMetaRequests
+ * can be handled in {@link TransportResponseHandler} a similar way.
+ *
+ * @since 3.2.0
+ */
+public interface BaseResponseCallback {
+
+  /** Exception either propagated from server or raised on client side. */
+  void onFailure(Throwable e);
+}

common/network-common/src/main/java/org/apache/spark/network/client/MergedBlockMetaResponseCallback.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.client;
+
+import org.apache.spark.network.buffer.ManagedBuffer;
+
+/**
+ * Callback for the result of a single
+ * {@link org.apache.spark.network.protocol.MergedBlockMetaRequest}.
+ *
+ * @since 3.2.0
+ */
+public interface MergedBlockMetaResponseCallback extends BaseResponseCallback {
+  /**
+   * Called upon receipt of a particular merged block meta.
+   *
+   * The given buffer will initially have a refcount of 1, but will be release()'d as soon as this
+   * call returns. You must therefore either retain() the buffer or copy its contents before
+   * returning.
+   *
+   * @param numChunks number of merged chunks in the merged block
+   * @param buffer the buffer contains an array of roaring bitmaps. The i-th roaring bitmap
+   *               contains the mapIds that were merged to the i-th merged chunk.
+   */
+  void onSuccess(int numChunks, ManagedBuffer buffer);
+}

common/network-common/src/main/java/org/apache/spark/network/client/RpcResponseCallback.java

@@ -23,15 +23,12 @@
  * Callback for the result of a single RPC. This will be invoked once with either success or
  * failure.
  */
-public interface RpcResponseCallback {
+public interface RpcResponseCallback extends BaseResponseCallback {
   /**
    * Successful serialized result from server.
    *
    * After `onSuccess` returns, `response` will be recycled and its content will become invalid.
    * Please copy the content of `response` if you want to use it after `onSuccess` returns.
    */
   void onSuccess(ByteBuffer response);
-
-  /** Exception either propagated from server or raised on client side. */
-  void onFailure(Throwable e);
 }