Update change_scala_version.py to also change scala.version property (#…

…9897)
dmlc · Dec 19, 2023 · 82d846b · 82d846b
1 parent 71d330a
commit 82d846b
Show file tree

Hide file tree

Showing 12 changed files with 89 additions and 35 deletions.
diff --git a/dev/change_scala_version.py b/dev/change_scala_version.py
@@ -3,37 +3,71 @@
 import re
 import shutil
 
-try:
-    import sh
-except ImportError as e:
-    raise ImportError(
-        "Please install sh in your Python environment.\n"
-        " - Pip: pip install sh\n"
-        " - Conda: conda install -c conda-forge sh"
-    ) from e
-
 
 def main(args):
+    if args.scala_version == "2.12":
+        scala_ver = "2.12"
+        scala_patchver = "2.12.18"
+    elif args.scala_version == "2.13":
+        scala_ver = "2.13"
+        scala_patchver = "2.13.11"
+    else:
+        raise ValueError(f"Unsupported Scala version: {args.scala_version}")
+
     # Clean artifacts
-    for target in pathlib.Path("jvm-packages/").glob("**/target"):
-        if target.is_dir():
-            print(f"Removing {target}...")
-            shutil.rmtree(target)
+    if args.purge_artifacts:
+        for target in pathlib.Path("jvm-packages/").glob("**/target"):
+            if target.is_dir():
+                print(f"Removing {target}...")
+                shutil.rmtree(target)
 
     # Update pom.xml
     for pom in pathlib.Path("jvm-packages/").glob("**/pom.xml"):
         print(f"Updating {pom}...")
-        sh.sed(
-            [
-                "-i",
-                f"s/<artifactId>xgboost-jvm_[0-9\\.]*/<artifactId>xgboost-jvm_{args.scala_version}/g",
-                str(pom),
-            ]
-        )
+        with open(pom, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+        with open(pom, "w", encoding="utf-8") as f:
+            replaced_scalaver = False
+            replaced_scala_binver = False
+            for line in lines:
+                for artifact in [
+                    "xgboost-jvm",
+                    "xgboost4j",
+                    "xgboost4j-gpu",
+                    "xgboost4j-spark",
+                    "xgboost4j-spark-gpu",
+                    "xgboost4j-flink",
+                    "xgboost4j-example",
+                ]:
+                    line = re.sub(
+                        f"<artifactId>{artifact}_[0-9\\.]*",
+                        f"<artifactId>{artifact}_{scala_ver}",
+                        line,
+                    )
+                # Only replace the first occurrence of scala.version
+                if not replaced_scalaver:
+                    line, nsubs = re.subn(
+                        r"<scala.version>[0-9\.]*",
+                        f"<scala.version>{scala_patchver}",
+                        line,
+                    )
+                    if nsubs > 0:
+                        replaced_scalaver = True
+                # Only replace the first occurrence of scala.binary.version
+                if not replaced_scala_binver:
+                    line, nsubs = re.subn(
+                        r"<scala.binary.version>[0-9\.]*",
+                        f"<scala.binary.version>{scala_ver}",
+                        line,
+                    )
+                    if nsubs > 0:
+                        replaced_scala_binver = True
+                f.write(line)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("--purge-artifacts", action="store_true")
     parser.add_argument(
         "--scala-version",
         type=str,

diff --git a/dev/prepare_jvm_release.py b/dev/prepare_jvm_release.py
@@ -229,7 +229,7 @@ def main():
     print(
         "5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n"
         "   export MAVEN_SKIP_NATIVE_BUILD=1\n"
-        "   python dev/change_scala_version.py --scala-version 2.13\n"
+        "   python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts\n"
         "   GPG_TTY=$(tty) mvn deploy -Prelease-cpu-only,scala-2.13 -DskipTests"
     )
     print(

diff --git a/jvm-packages/xgboost4j-example/pom.xml b/jvm-packages/xgboost4j-example/pom.xml
@@ -9,7 +9,7 @@
         <version>2.0.3</version>
     </parent>
     <name>xgboost4j-example</name>
-    <artifactId>xgboost4j-example_${scala.binary.version}</artifactId>
+    <artifactId>xgboost4j-example_2.12</artifactId>
     <version>2.0.3</version>
     <packaging>jar</packaging>
     <build>
@@ -26,7 +26,7 @@
     <dependencies>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j-spark_${scala.binary.version}</artifactId>
+            <artifactId>xgboost4j-spark_2.12</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
@@ -37,7 +37,7 @@
         </dependency>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j-flink_${scala.binary.version}</artifactId>
+            <artifactId>xgboost4j-flink_2.12</artifactId>
             <version>${project.version}</version>
         </dependency>
     </dependencies>

diff --git a/jvm-packages/xgboost4j-flink/pom.xml b/jvm-packages/xgboost4j-flink/pom.xml
@@ -10,7 +10,7 @@
     </parent>
 
     <name>xgboost4j-flink</name>
-    <artifactId>xgboost4j-flink_${scala.binary.version}</artifactId>
+    <artifactId>xgboost4j-flink_2.12</artifactId>
     <version>2.0.3</version>
     <properties>
       <flink-ml.version>2.2.0</flink-ml.version>
@@ -30,7 +30,7 @@
     <dependencies>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j_${scala.binary.version}</artifactId>
+            <artifactId>xgboost4j_2.12</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

diff --git a/jvm-packages/xgboost4j-gpu/pom.xml b/jvm-packages/xgboost4j-gpu/pom.xml
@@ -8,7 +8,7 @@
         <artifactId>xgboost-jvm_2.12</artifactId>
         <version>2.0.3</version>
     </parent>
-    <artifactId>xgboost4j-gpu_${scala.binary.version}</artifactId>
+    <artifactId>xgboost4j-gpu_2.12</artifactId>
     <name>xgboost4j-gpu</name>
     <version>2.0.3</version>
     <packaging>jar</packaging>

diff --git a/jvm-packages/xgboost4j-spark-gpu/pom.xml b/jvm-packages/xgboost4j-spark-gpu/pom.xml
@@ -9,7 +9,7 @@
         <version>2.0.3</version>
     </parent>
     <name>xgboost4j-spark-gpu</name>
-    <artifactId>xgboost4j-spark-gpu_${scala.binary.version}</artifactId>
+    <artifactId>xgboost4j-spark-gpu_2.12</artifactId>
     <build>
         <plugins>
             <plugin>
@@ -24,7 +24,7 @@
     <dependencies>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j-gpu_${scala.binary.version}</artifactId>
+            <artifactId>xgboost4j-gpu_2.12</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

diff --git a/jvm-packages/xgboost4j-spark/pom.xml b/jvm-packages/xgboost4j-spark/pom.xml
@@ -9,7 +9,7 @@
         <version>2.0.3</version>
     </parent>
     <name>xgboost4j-spark</name>
-    <artifactId>xgboost4j-spark_${scala.binary.version}</artifactId>
+    <artifactId>xgboost4j-spark_2.12</artifactId>
     <build>
         <plugins>
             <plugin>
@@ -24,7 +24,7 @@
     <dependencies>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j_${scala.binary.version}</artifactId>
+            <artifactId>xgboost4j_2.12</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml
@@ -9,7 +9,7 @@
         <version>2.0.3</version>
     </parent>
     <name>xgboost4j</name>
-    <artifactId>xgboost4j_${scala.binary.version}</artifactId>
+    <artifactId>xgboost4j_2.12</artifactId>
     <version>2.0.3</version>
     <packaging>jar</packaging>
 

diff --git a/tests/buildkite/build-jvm-packages.sh b/tests/buildkite/build-jvm-packages.sh
@@ -8,13 +8,18 @@ echo "--- Build XGBoost JVM packages scala 2.12"
 tests/ci_build/ci_build.sh jvm docker tests/ci_build/build_jvm_packages.sh \
   ${SPARK_VERSION}
 
+echo "--- Stash XGBoost4J JARs (Scala 2.12)"
+buildkite-agent artifact upload "jvm-packages/xgboost4j/target/*.jar"
+buildkite-agent artifact upload "jvm-packages/xgboost4j-spark/target/*.jar"
+buildkite-agent artifact upload "jvm-packages/xgboost4j-flink/target/*.jar"
+buildkite-agent artifact upload "jvm-packages/xgboost4j-example/target/*.jar"
 
 echo "--- Build XGBoost JVM packages scala 2.13"
 
 tests/ci_build/ci_build.sh jvm docker tests/ci_build/build_jvm_packages.sh \
   ${SPARK_VERSION} "" "" "true"
 
-echo "--- Stash XGBoost4J JARs"
+echo "--- Stash XGBoost4J JARs (Scala 2.13)"
 buildkite-agent artifact upload "jvm-packages/xgboost4j/target/*.jar"
 buildkite-agent artifact upload "jvm-packages/xgboost4j-spark/target/*.jar"
 buildkite-agent artifact upload "jvm-packages/xgboost4j-flink/target/*.jar"

diff --git a/tests/ci_build/build_jvm_packages.sh b/tests/ci_build/build_jvm_packages.sh
@@ -27,6 +27,9 @@ fi
 mvn_profile_string=""
 if [ "x$use_scala213" != "x" ]; then
   export mvn_profile_string="-Pdefault,scala-2.13"
+  cd ..
+  python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts
+  cd jvm-packages
 fi
 
 mvn --no-transfer-progress package $mvn_profile_string -Dspark.version=${spark_version} $gpu_options

diff --git a/tests/ci_build/deploy_jvm_packages.sh b/tests/ci_build/deploy_jvm_packages.sh
@@ -27,6 +27,9 @@ rm -rf ../build/
 # Deploy to S3 bucket xgboost-maven-repo
 mvn --no-transfer-progress package deploy -P default,gpu,release-to-s3 -Dspark.version=${spark_version} -DskipTests
 # Deploy scala 2.13 to S3 bucket xgboost-maven-repo
+cd ..
+python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts
+cd jvm-packages/
 mvn --no-transfer-progress package deploy -P release-to-s3,default,scala-2.13 -Dspark.version=${spark_version} -DskipTests
 
 

diff --git a/tests/ci_build/test_jvm_cross.sh b/tests/ci_build/test_jvm_cross.sh
@@ -21,9 +21,18 @@ if [ ! -z "$RUN_INTEGRATION_TEST" ]; then
 fi
 
 # including maven profiles for different scala versions: 2.12 is the default at the moment.
-for _maven_profile_string in "" "-Pdefault,scala-2.13"; do
+for scala_binary_version in "2.12" "2.13"; do
+  cd ..
+  python dev/change_scala_version.py --scala-version ${scala_binary_version}
+  cd jvm-packages
   scala_version=$(mvn help:evaluate $_maven_profile_string -Dexpression=scala.version -q -DforceStdout)
-  scala_binary_version=$(mvn help:evaluate $_maven_profile_string -Dexpression=scala.binary.version -q -DforceStdout)
+  if [[ "$scala_binary_version" == "2.12" ]]; then
+    _maven_profile_string=""
+  elif [[ "$scala_binary_version" == "2.13" ]]; then
+    _maven_profile_string="-Pdefault,scala-2.13"
+  else
+    echo "Unexpected scala version: $scala_version ($scala_binary_version)."
+  fi
 
   # Install XGBoost4J JAR into local Maven repository
   mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar