[SPARK-44113][BUILD][INFRA][DOCS] Drop support for Scala 2.12

.github/workflows/benchmark.yml

@@ -31,9 +31,9 @@ on:
         required: true
         default: '8'
       scala:
-        description: 'Scala version: 2.12 or 2.13'
+        description: 'Scala version: 2.13'
         required: true
-        default: '2.12'
+        default: '2.13'
       failfast:
         description: 'Failfast: true or false'
         required: true
@@ -170,7 +170,6 @@ jobs:
         key: tpcds-${{ hashFiles('.github/workflows/benchmark.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }}
     - name: Run benchmarks
       run: |
-        dev/change-scala-version.sh ${{ github.event.inputs.scala }}
         ./build/sbt -Pscala-${{ github.event.inputs.scala }} -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pspark-ganglia-lgpl Test/package
         # Make less noisy
         cp conf/log4j2.properties.template conf/log4j2.properties
@@ -181,8 +180,6 @@ jobs:
           --jars "`find . -name '*-SNAPSHOT-tests.jar' -o -name '*avro*-SNAPSHOT.jar' | paste -sd ',' -`" \
           "`find . -name 'spark-core*-SNAPSHOT-tests.jar'`" \
           "${{ github.event.inputs.class }}"
-        # Revert to default Scala version to clean up unnecessary git diff
-        dev/change-scala-version.sh 2.12
         # To keep the directory structure and file permissions, tar them
         # See also https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files
         echo "Preparing the benchmark results:"

.github/workflows/build_and_test.yml

@@ -86,7 +86,7 @@ jobs:
           sparkr=`./dev/is-changed.py -m sparkr`
           tpcds=`./dev/is-changed.py -m sql`
           docker=`./dev/is-changed.py -m docker-integration-tests`
-          # 'build', 'scala-213', and 'java-other-versions' are always true for now.
+          # 'build' and 'java-other-versions' are always true for now.
           # It does not save significant time and most of PRs trigger the build.
           precondition="
             {
@@ -95,7 +95,6 @@ jobs:
               \"sparkr\": \"$sparkr\",
               \"tpcds-1g\": \"$tpcds\",
               \"docker-integration-tests\": \"$docker\",
-              \"scala-213\": \"true\",
               \"java-other-versions\": \"true\",
               \"lint\" : \"true\",
               \"k8s-integration-tests\" : \"true\",
@@ -828,53 +827,6 @@ jobs:
         ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} install
         rm -rf ~/.m2/repository/org/apache/spark
 
-  scala-213:
-    needs: precondition
-    if: fromJson(needs.precondition.outputs.required).scala-213 == 'true'
-    name: Scala 2.13 build with SBT
-    runs-on: ubuntu-22.04
-    timeout-minutes: 300
-    steps:
-    - name: Checkout Spark repository
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        repository: apache/spark
-        ref: ${{ inputs.branch }}
-    - name: Sync the current branch with the latest in Apache Spark
-      if: github.repository != 'apache/spark'
-      run: |
-        git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
-        git -c user.name='Apache Spark Test Account' -c user.email='[email protected]' merge --no-commit --progress --squash FETCH_HEAD
-        git -c user.name='Apache Spark Test Account' -c user.email='[email protected]' commit -m "Merged commit" --allow-empty
-    - name: Cache Scala, SBT and Maven
-      uses: actions/cache@v3
-      with:
-        path: |
-          build/apache-maven-*
-          build/scala-*
-          build/*.jar
-          ~/.sbt
-        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
-        restore-keys: |
-          build-
-    - name: Cache Coursier local repository
-      uses: actions/cache@v3
-      with:
-        path: ~/.cache/coursier
-        key: scala-213-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
-        restore-keys: |
-          scala-213-coursier-
-    - name: Install Java 8
-      uses: actions/setup-java@v3
-      with:
-        distribution: zulu
-        java-version: 8
-    - name: Build with SBT
-      run: |
-        ./dev/change-scala-version.sh 2.13
-        ./build/sbt -Pyarn -Pmesos -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pdocker-integration-tests -Pkubernetes-integration-tests -Pspark-ganglia-lgpl -Pscala-2.13 compile Test/compile
-
   # Any TPC-DS related updates on this job need to be applied to tpcds-1g-gen job of benchmark.yml as well
   tpcds-1g:
     needs: precondition

.github/workflows/build_ansi.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / ANSI (master, Hadoop 3, JDK 8, Scala 2.12)"
+name: "Build / ANSI (master, Hadoop 3, JDK 8, Scala 2.13)"
 
 on:
   schedule:

.github/workflows/build_coverage.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Coverage (master, Scala 2.12, Hadoop 3, JDK 8)"
+name: "Build / Coverage (master, Scala 2.13, Hadoop 3, JDK 8)"
 
 on:
   schedule:

.github/workflows/build_java11.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build (master, Scala 2.12, Hadoop 3, JDK 11)"
+name: "Build (master, Scala 2.13, Hadoop 3, JDK 11)"
 
 on:
   schedule:

.github/workflows/build_java17.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build (master, Scala 2.12, Hadoop 3, JDK 17)"
+name: "Build (master, Scala 2.13, Hadoop 3, JDK 17)"
 
 on:
   schedule:

.github/workflows/build_java21.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build (master, Scala 2.12, Hadoop 3, JDK 21)"
+name: "Build (master, Scala 2.13, Hadoop 3, JDK 21)"
 
 on:
   schedule:

.github/workflows/build_maven.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build using Maven (master, Scala 2.12, Hadoop 3, JDK 8)"
+name: "Build using Maven (master, Scala 2.13, Hadoop 3, JDK 8)"
 
 on:
   schedule:

.github/workflows/build_rockdb_as_ui_backend.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / RocksDB as UI Backend (master, Hadoop 3, JDK 8, Scala 2.12)"
+name: "Build / RocksDB as UI Backend (master, Hadoop 3, JDK 8, Scala 2.13)"
 
 on:
   schedule:

LICENSE-binary

@@ -209,34 +209,34 @@ org.apache.zookeeper:zookeeper
 oro:oro
 commons-configuration:commons-configuration
 commons-digester:commons-digester
-com.chuusai:shapeless_2.12
+com.chuusai:shapeless_2.13
 com.googlecode.javaewah:JavaEWAH
 com.twitter:chill-java
-com.twitter:chill_2.12
+com.twitter:chill_2.13
 com.univocity:univocity-parsers
 javax.jdo:jdo-api
 joda-time:joda-time
 net.sf.opencsv:opencsv
 org.apache.derby:derby
 org.objenesis:objenesis
 org.roaringbitmap:RoaringBitmap
-org.scalanlp:breeze-macros_2.12
-org.scalanlp:breeze_2.12
-org.typelevel:macro-compat_2.12
+org.scalanlp:breeze-macros_2.13
+org.scalanlp:breeze_2.13
+org.typelevel:macro-compat_2.13
 org.yaml:snakeyaml
 org.apache.xbean:xbean-asm7-shaded
 com.squareup.okhttp3:logging-interceptor
 com.squareup.okhttp3:okhttp
 com.squareup.okio:okio
-org.apache.spark:spark-catalyst_2.12
-org.apache.spark:spark-kvstore_2.12
-org.apache.spark:spark-launcher_2.12
-org.apache.spark:spark-mllib-local_2.12
-org.apache.spark:spark-network-common_2.12
-org.apache.spark:spark-network-shuffle_2.12
-org.apache.spark:spark-sketch_2.12
-org.apache.spark:spark-tags_2.12
-org.apache.spark:spark-unsafe_2.12
+org.apache.spark:spark-catalyst_2.13
+org.apache.spark:spark-kvstore_2.13
+org.apache.spark:spark-launcher_2.13
+org.apache.spark:spark-mllib-local_2.13
+org.apache.spark:spark-network-common_2.13
+org.apache.spark:spark-network-shuffle_2.13
+org.apache.spark:spark-sketch_2.13
+org.apache.spark:spark-tags_2.13
+org.apache.spark:spark-unsafe_2.13
 commons-httpclient:commons-httpclient
 com.vlkan:flatbuffers
 com.ning:compress-lzf
@@ -299,10 +299,10 @@ org.apache.orc:orc-mapreduce
 org.mortbay.jetty:jetty
 org.mortbay.jetty:jetty-util
 com.jolbox:bonecp
-org.json4s:json4s-ast_2.12
-org.json4s:json4s-core_2.12
-org.json4s:json4s-jackson_2.12
-org.json4s:json4s-scalap_2.12
+org.json4s:json4s-ast_2.13
+org.json4s:json4s-core_2.13
+org.json4s:json4s-jackson_2.13
+org.json4s:json4s-scalap_2.13
 com.carrotsearch:hppc
 com.fasterxml.jackson.core:jackson-annotations
 com.fasterxml.jackson.core:jackson-core
@@ -312,7 +312,7 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-base
 com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
 com.fasterxml.jackson.module:jackson-module-jaxb-annotations
 com.fasterxml.jackson.module:jackson-module-paranamer
-com.fasterxml.jackson.module:jackson-module-scala_2.12
+com.fasterxml.jackson.module:jackson-module-scala_2.13
 com.github.mifmif:generex
 com.google.code.findbugs:jsr305
 com.google.code.gson:gson
@@ -385,8 +385,8 @@ org.eclipse.jetty:jetty-xml
 org.scala-lang:scala-compiler
 org.scala-lang:scala-library
 org.scala-lang:scala-reflect
-org.scala-lang.modules:scala-parser-combinators_2.12
-org.scala-lang.modules:scala-xml_2.12
+org.scala-lang.modules:scala-parser-combinators_2.13
+org.scala-lang.modules:scala-xml_2.13
 com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter
 com.zaxxer.HikariCP
 org.apache.hive:hive-beeline
@@ -471,19 +471,19 @@ MIT License
 -----------
 
 com.microsoft.sqlserver:mssql-jdbc
-org.typelevel:spire_2.12
-org.typelevel:spire-macros_2.12
-org.typelevel:spire-platform_2.12
-org.typelevel:spire-util_2.12
-org.typelevel:algebra_2.12:jar
-org.typelevel:cats-kernel_2.12
-org.typelevel:machinist_2.12
+org.typelevel:spire_2.13
+org.typelevel:spire-macros_2.13
+org.typelevel:spire-platform_2.13
+org.typelevel:spire-util_2.13
+org.typelevel:algebra_2.13:jar
+org.typelevel:cats-kernel_2.13
+org.typelevel:machinist_2.13
 net.razorvine:pickle
 org.slf4j:jcl-over-slf4j
 org.slf4j:jul-to-slf4j
 org.slf4j:slf4j-api
 org.slf4j:slf4j-log4j12
-com.github.scopt:scopt_2.12
+com.github.scopt:scopt_2.13
 dev.ludovic.netlib:blas
 dev.ludovic.netlib:arpack
 dev.ludovic.netlib:lapack

assembly/pom.xml

@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>4.0.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-assembly_2.12</artifactId>
+  <artifactId>spark-assembly_2.13</artifactId>
   <name>Spark Project Assembly</name>
   <url>https://spark.apache.org/</url>
   <packaging>pom</packaging>

bin/load-spark-env.cmd

@@ -39,21 +39,21 @@ set SCALA_VERSION_2=2.12
 set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_1%"
 set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_2%"
 set ENV_VARIABLE_DOC=https://spark.apache.org/docs/latest/configuration.html#environment-variables
-
-if not defined SPARK_SCALA_VERSION (
-  if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% (
-    echo Presence of build for multiple Scala versions detected ^(%ASSEMBLY_DIR1% and %ASSEMBLY_DIR2%^).
-    echo Remove one of them or, set SPARK_SCALA_VERSION=%SCALA_VERSION_1% in spark-env.cmd.
-    echo Visit %ENV_VARIABLE_DOC% for more details about setting environment variables in spark-env.cmd.
-    echo Either clean one of them or, set SPARK_SCALA_VERSION in spark-env.cmd.
-    exit 1
-  )
-  if exist %ASSEMBLY_DIR1% (
-    set SPARK_SCALA_VERSION=%SCALA_VERSION_1%
-  ) else (
-    set SPARK_SCALA_VERSION=%SCALA_VERSION_2%
-  )
-)
+set SPARK_SCALA_VERSION=2.13
+rem if not defined SPARK_SCALA_VERSION (
+rem   if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% (
+rem     echo Presence of build for multiple Scala versions detected ^(%ASSEMBLY_DIR1% and %ASSEMBLY_DIR2%^).
+rem     echo Remove one of them or, set SPARK_SCALA_VERSION=%SCALA_VERSION_1% in spark-env.cmd.
+rem     echo Visit %ENV_VARIABLE_DOC% for more details about setting environment variables in spark-env.cmd.
+rem     echo Either clean one of them or, set SPARK_SCALA_VERSION in spark-env.cmd.
+rem     exit 1
+rem   )
+rem   if exist %ASSEMBLY_DIR1% (
+rem     set SPARK_SCALA_VERSION=%SCALA_VERSION_1%
+rem   ) else (
+rem     set SPARK_SCALA_VERSION=%SCALA_VERSION_2%
+rem   )
+rem )
 exit /b 0
 
 :LoadSparkEnv

bin/load-spark-env.sh

@@ -42,27 +42,27 @@ if [ -z "$SPARK_ENV_LOADED" ]; then
 fi
 
 # Setting SPARK_SCALA_VERSION if not already set.
-
-if [ -z "$SPARK_SCALA_VERSION" ]; then
-  SCALA_VERSION_1=2.13
-  SCALA_VERSION_2=2.12
-
-  ASSEMBLY_DIR_1="${SPARK_HOME}/assembly/target/scala-${SCALA_VERSION_1}"
-  ASSEMBLY_DIR_2="${SPARK_HOME}/assembly/target/scala-${SCALA_VERSION_2}"
-  ENV_VARIABLE_DOC="https://spark.apache.org/docs/latest/configuration.html#environment-variables"
-  if [[ -d "$ASSEMBLY_DIR_1" && -d "$ASSEMBLY_DIR_2" ]]; then
-    echo "Presence of build for multiple Scala versions detected ($ASSEMBLY_DIR_1 and $ASSEMBLY_DIR_2)." 1>&2
-    echo "Remove one of them or, export SPARK_SCALA_VERSION=$SCALA_VERSION_1 in ${SPARK_ENV_SH}." 1>&2
-    echo "Visit ${ENV_VARIABLE_DOC} for more details about setting environment variables in spark-env.sh." 1>&2
-    exit 1
-  fi
-
-  if [[ -d "$ASSEMBLY_DIR_1" ]]; then
-    export SPARK_SCALA_VERSION=${SCALA_VERSION_1}
-  else
-    export SPARK_SCALA_VERSION=${SCALA_VERSION_2}
-  fi
-fi
+export SPARK_SCALA_VERSION=2.13
+#if [ -z "$SPARK_SCALA_VERSION" ]; then
+#  SCALA_VERSION_1=2.13
+#  SCALA_VERSION_2=2.12
+#
+#  ASSEMBLY_DIR_1="${SPARK_HOME}/assembly/target/scala-${SCALA_VERSION_1}"
+#  ASSEMBLY_DIR_2="${SPARK_HOME}/assembly/target/scala-${SCALA_VERSION_2}"
+#  ENV_VARIABLE_DOC="https://spark.apache.org/docs/latest/configuration.html#environment-variables"
+#  if [[ -d "$ASSEMBLY_DIR_1" && -d "$ASSEMBLY_DIR_2" ]]; then
+#    echo "Presence of build for multiple Scala versions detected ($ASSEMBLY_DIR_1 and $ASSEMBLY_DIR_2)." 1>&2
+#    echo "Remove one of them or, export SPARK_SCALA_VERSION=$SCALA_VERSION_1 in ${SPARK_ENV_SH}." 1>&2
+#    echo "Visit ${ENV_VARIABLE_DOC} for more details about setting environment variables in spark-env.sh." 1>&2
+#    exit 1
+#  fi
+#
+#  if [[ -d "$ASSEMBLY_DIR_1" ]]; then
+#    export SPARK_SCALA_VERSION=${SCALA_VERSION_1}
+#  else
+#    export SPARK_SCALA_VERSION=${SCALA_VERSION_2}
+#  fi
+#fi
 
 # Append jline option to enable the Beeline process to run in background.
 if [[ ( ! $(ps -o stat= -p $$) =~ "+" ) && ! ( -p /dev/stdin ) ]]; then

common/kvstore/pom.xml

@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.12</artifactId>
+    <artifactId>spark-parent_2.13</artifactId>
     <version>4.0.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-kvstore_2.12</artifactId>
+  <artifactId>spark-kvstore_2.13</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Local DB</name>
   <url>https://spark.apache.org/</url>