huangxiaopingRD · pull · Dec 23, 2023 · Dec 21, 2023 · Dec 21, 2023 · Dec 21, 2023
diff --git a/common/utils/src/main/resources/error/error-classes.json b/common/utils/src/main/resources/error/error-classes.json
@@ -2513,6 +2513,12 @@
     ],
     "sqlState" : "42601"
   },
+  "INVALID_WRITER_COMMIT_MESSAGE" : {
+    "message" : [
+      "The data source writer has generated an invalid number of commit messages. Expected exactly one writer commit message from each task, but received <detail>."
+    ],
+    "sqlState" : "42KDE"
+  },
   "INVALID_WRITE_DISTRIBUTION" : {
     "message" : [
       "The requested write distribution is invalid."

diff --git a/.../org/apache/spark/util/IvyTestUtils.scala → .../org/apache/spark/util/IvyTestUtils.scala b/.../org/apache/spark/util/IvyTestUtils.scala → .../org/apache/spark/util/IvyTestUtils.scala
diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml
@@ -82,6 +82,13 @@
       <artifactId>scalacheck_${scala.binary.version}</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-common-utils_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
     <!-- Use mima to perform the compatibility check -->
     <dependency>
       <groupId>com.typesafe</groupId>

diff --git a/core/pom.xml b/core/pom.xml
@@ -451,6 +451,13 @@
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-common-utils_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
 
     <!--
       This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude

diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -194,6 +194,8 @@ class BarrierTaskContext private[spark] (
 
   override def isCompleted(): Boolean = taskContext.isCompleted()
 
+  override def isFailed(): Boolean = taskContext.isFailed()
+
   override def isInterrupted(): Boolean = taskContext.isInterrupted()
 
   override def addTaskCompletionListener(listener: TaskCompletionListener): this.type = {

diff --git a/core/src/main/scala/org/apache/spark/TaskContext.scala b/core/src/main/scala/org/apache/spark/TaskContext.scala
@@ -94,6 +94,11 @@ abstract class TaskContext extends Serializable {
    */
   def isCompleted(): Boolean
 
+  /**
+   * Returns true if the task has failed.
+   */
+  def isFailed(): Boolean
+
   /**
    * Returns true if the task has been killed.
    */

diff --git a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
@@ -275,6 +275,8 @@ private[spark] class TaskContextImpl(
   @GuardedBy("this")
   override def isCompleted(): Boolean = synchronized(completed)
 
+  override def isFailed(): Boolean = synchronized(failureCauseOpt.isDefined)
+
   override def isInterrupted(): Boolean = reasonIfKilled.isDefined
 
   override def getLocalProperty(key: String): String = localProperties.getProperty(key)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala
@@ -670,6 +670,16 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark
     assert(invocationOrder === Seq("C", "B", "A", "D"))
   }
 
+  test("SPARK-46480: Add isFailed in TaskContext") {
+    val context = TaskContext.empty()
+    var isFailed = false
+    context.addTaskCompletionListener[Unit] { context =>
+      isFailed = context.isFailed()
+    }
+    context.markTaskFailed(new RuntimeException())
+    context.markTaskCompleted(None)
+    assert(isFailed)
+  }
 }
 
 private object TaskContextSuite {

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
@@ -770,7 +770,7 @@ def __hash__(self):
         "pyspark.pandas.tests.window.test_groupby_rolling_adv",
         "pyspark.pandas.tests.window.test_groupby_rolling_count",
         "pyspark.pandas.tests.test_scalars",
-        "pyspark.pandas.tests.test_series_datetime",
+        "pyspark.pandas.tests.series.test_datetime",
         "pyspark.pandas.tests.series.test_string_ops_adv",
         "pyspark.pandas.tests.series.test_string_ops_basic",
         "pyspark.pandas.tests.test_spark_functions",
@@ -864,6 +864,12 @@ def __hash__(self):
         "pyspark.pandas.tests.test_indexing",
         "pyspark.pandas.tests.test_ops_on_diff_frames",
         "pyspark.pandas.tests.diff_frames_ops.test_align",
+        "pyspark.pandas.tests.diff_frames_ops.test_arithmetic",
+        "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_ext",
+        "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_ext_float",
+        "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_chain",
+        "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_chain_ext",
+        "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_chain_ext_float",
         "pyspark.pandas.tests.diff_frames_ops.test_basic_slow",
         "pyspark.pandas.tests.diff_frames_ops.test_cov",
         "pyspark.pandas.tests.diff_frames_ops.test_corrwith",
@@ -1046,7 +1052,7 @@ def __hash__(self):
         "pyspark.pandas.tests.connect.resample.test_parity_on",
         "pyspark.pandas.tests.connect.resample.test_parity_timezone",
         "pyspark.pandas.tests.connect.test_parity_scalars",
-        "pyspark.pandas.tests.connect.test_parity_series_datetime",
+        "pyspark.pandas.tests.connect.series.test_parity_datetime",
         "pyspark.pandas.tests.connect.series.test_parity_string_ops_adv",
         "pyspark.pandas.tests.connect.series.test_parity_string_ops_basic",
         "pyspark.pandas.tests.connect.test_parity_spark_functions",
@@ -1223,6 +1229,12 @@ def __hash__(self):
         "pyspark.pandas.tests.connect.indexes.test_parity_datetime_property",
         "pyspark.pandas.tests.connect.indexes.test_parity_datetime_round",
         "pyspark.pandas.tests.connect.test_parity_ops_on_diff_frames",
+        "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic",
+        "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_ext",
+        "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_ext_float",
+        "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_chain",
+        "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_chain_ext",
+        "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_chain_ext_float",
         "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby",
         "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_aggregate",
         "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_apply",

diff --git a/docs/.gitignore b/docs/.gitignore
@@ -1 +1,2 @@
 generated-*.html
+_generated_function_html/
diff --git a/docs/css/custom.css b/docs/css/custom.css
@@ -7,7 +7,7 @@ body {
   font-style: normal;
   font-weight: 400;
   overflow-wrap: anywhere;
-  overflow-x: hidden;
+  overflow-x: auto;
   padding-top: 80px;
   padding-bottom: 20px;
 }

diff --git a/docs/sql-error-conditions.md b/docs/sql-error-conditions.md
@@ -1398,6 +1398,12 @@ Rewrite the query to avoid window functions, aggregate functions, and generator
 
 Cannot specify ORDER BY or a window frame for `<aggFunc>`.
 
+### INVALID_WRITER_COMMIT_MESSAGE
+
+[SQLSTATE: 42KDE](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation)
+
+The data source writer has generated an invalid number of commit messages. Expected exactly one writer commit message from each task, but received `<detail>`.
+
 ### [INVALID_WRITE_DISTRIBUTION](sql-error-conditions-invalid-write-distribution-error-class.html)
 
 [SQLSTATE: 42000](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation)