diff --git a/celeborn-0.6.yaml b/celeborn-0.6.yaml new file mode 100644 index 00000000000..e27c9a0dff2 --- /dev/null +++ b/celeborn-0.6.yaml @@ -0,0 +1,168 @@ +package: + name: celeborn-0.6 + version: "0.6.1" + epoch: 0 # GHSA-jq43-27x9-3v86 + description: "Apache Celeborn - A Remote Shuffle Service for Distributed Data Processing Engines" + copyright: + - license: Apache-2.0 + dependencies: + provides: + - celeborn=${{package.full-version}} + runtime: + - bash + - coreutils + - glibc-locale-en + - libjemalloc2 + - openjdk-${{vars.java-version}}-default-jvm + - openssh + - procps + - sed + - tzdata + +vars: + base-package-name: celeborn + java-version: 8 + +environment: + contents: + packages: + - bash + - busybox + - git + - libstdc++ + - maven + - openjdk-${{vars.java-version}}-default-jdk + environment: + JAVA_HOME: /usr/lib/jvm/java-1.${{vars.java-version}}-openjdk + LANG: en_US.UTF-8 + +pipeline: + - uses: git-checkout + with: + expected-commit: 29d03860c0f219eec68854e2c8535b14c899fdfa + repository: https://github.com/apache/celeborn.git + tag: v${{package.version}} + + - uses: maven/pombump + + - uses: patch + with: + patches: commons-lang3-3.18.0.patch + + - name: Build / install Celeborn + runs: | + # Build parts taken from https://github.com/apache/celeborn/blob/main/build/make-distribution.sh + + dest=${{targets.contextdir}}/usr/share/java/${{vars.base-package-name}} + mkdir -p \ + $dest \ + $dest/bin \ + $dest/conf \ + $dest/jars \ + $dest/sbin + + SCALA_VERSION="$(mvn help:evaluate -Dexpression=scala.binary.version 2>/dev/null \ + | grep -v "INFO" \ + | grep -v "WARNING" \ + | tail -n 1)" + VERSION="$(mvn help:evaluate -Dexpression=project.version 2>/dev/null \ + | grep -v "INFO" \ + | grep -v "WARNING" \ + | tail -n 1)" + + echo "Celeborn ${{package.version}} (git revision $(git rev-parse --short HEAD 2>/dev/null))" > $dest/RELEASE + + mvn clean package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip -pl master,worker -am + + for dir in master worker + do + mkdir -p "$dest/$dir-jars" + + cp "$dir"/target/celeborn-"$dir"_"$SCALA_VERSION"-"$VERSION".jar "$dest/$dir-jars/" + cp "$dir"/target/scala-"$SCALA_VERSION"/jars/*.jar "$dest/jars/" + + for jar in "$dir/target/scala-$SCALA_VERSION/jars"/*; + do + (cd "$dest/$dir-jars"; ln -snf "../jars/$(basename $jar)" .) + done + done + + cp conf/*.template $dest/conf + cp -r bin $dest + cp -r sbin $dest + +subpackages: + - name: ${{package.name}}-compat + description: Compat package for ${{package.name}} + dependencies: + provides: + - ${{vars.base-package-name}}-compat=${{package.full-version}} + pipeline: + - runs: | + mkdir -p ${{targets.contextdir}}/opt/${{vars.base-package-name}} + + for dir in bin conf jars master-jars sbin worker-jars + do + ln -sf /usr/share/java/${{vars.base-package-name}}/$dir ${{targets.contextdir}}/opt/${{vars.base-package-name}}/$dir + done + test: + environment: + contents: + packages: + - ${{package.name}} + pipeline: + - name: Test symlinks + runs: | + for dir in bin conf jars master-jars sbin worker-jars + do + stat -L /opt/${{vars.base-package-name}}/$dir + done + +test: + environment: + contents: + packages: + - ${{package.name}}-compat + - wait-for-it + # Some of these are set to match defaults in the external counterpart image + environment: + CELEBORN_HOME: /opt/celeborn + CELEBORN_JEMALLOC_PATH: /usr/lib/libjemalloc.so.2 + CELEBORN_PREFER_JEMALLOC: true + JAVA_HOME: /usr/lib/jvm/java-1.${{vars.java-version}}-openjdk + LC_ALL: en_US.UTF-8 + pipeline: + - name: Test starting master + runs: | + cd $CELEBORN_HOME/conf + cp log4j2.xml.template log4j2.xml + + cat >> celeborn-defaults.conf < +Date: Wed, 23 Oct 2024 09:23:28 -0400 +Subject: [PATCH] Bump commons-lang3 from 3.17.0 to 3.18.0 + +--- + dev/deps/dependencies-client-flink-1.16 | 2 +- + dev/deps/dependencies-client-flink-1.17 | 2 +- + dev/deps/dependencies-client-flink-1.18 | 2 +- + dev/deps/dependencies-client-flink-1.19 | 2 +- + dev/deps/dependencies-client-flink-1.20 | 2 +- + dev/deps/dependencies-client-flink-2.0 | 2 +- + dev/deps/dependencies-client-mr | 2 +- + dev/deps/dependencies-client-spark-2.4 | 2 +- + dev/deps/dependencies-client-spark-3.0 | 2 +- + dev/deps/dependencies-client-spark-3.1 | 2 +- + dev/deps/dependencies-client-spark-3.2 | 2 +- + dev/deps/dependencies-client-spark-3.3 | 2 +- + dev/deps/dependencies-client-spark-3.4 | 2 +- + dev/deps/dependencies-client-spark-3.5 | 2 +- + dev/deps/dependencies-client-spark-4.0 | 2 +- + dev/deps/dependencies-client-tez | 2 +- + dev/deps/dependencies-server | 2 +- + project/CelebornBuild.scala | 2 +- + 18 files changed, 18 insertions(+), 18 deletions(-) + +diff --git a/dev/deps/dependencies-client-flink-1.16 b/dev/deps/dependencies-client-flink-1.16 +index 9c2fc4476..aa14aa42e 100644 +--- a/dev/deps/dependencies-client-flink-1.16 ++++ b/dev/deps/dependencies-client-flink-1.16 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-flink-1.17 b/dev/deps/dependencies-client-flink-1.17 +index 9c2fc4476..aa14aa42e 100644 +--- a/dev/deps/dependencies-client-flink-1.17 ++++ b/dev/deps/dependencies-client-flink-1.17 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-flink-1.18 b/dev/deps/dependencies-client-flink-1.18 +index 9c2fc4476..aa14aa42e 100644 +--- a/dev/deps/dependencies-client-flink-1.18 ++++ b/dev/deps/dependencies-client-flink-1.18 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-flink-1.19 b/dev/deps/dependencies-client-flink-1.19 +index 9c2fc4476..aa14aa42e 100644 +--- a/dev/deps/dependencies-client-flink-1.19 ++++ b/dev/deps/dependencies-client-flink-1.19 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-flink-1.20 b/dev/deps/dependencies-client-flink-1.20 +index 9c2fc4476..aa14aa42e 100644 +--- a/dev/deps/dependencies-client-flink-1.20 ++++ b/dev/deps/dependencies-client-flink-1.20 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-flink-2.0 b/dev/deps/dependencies-client-flink-2.0 +index 9c2fc4476..aa14aa42e 100644 +--- a/dev/deps/dependencies-client-flink-2.0 ++++ b/dev/deps/dependencies-client-flink-2.0 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr +index 82dc2cb11..9f4b04a3f 100644 +--- a/dev/deps/dependencies-client-mr ++++ b/dev/deps/dependencies-client-mr +@@ -31,7 +31,7 @@ commons-configuration2/2.8.0//commons-configuration2-2.8.0.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-daemon/1.0.13//commons-daemon-1.0.13.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + commons-math3/3.1.1//commons-math3-3.1.1.jar + commons-net/3.9.0//commons-net-3.9.0.jar +diff --git a/dev/deps/dependencies-client-spark-2.4 b/dev/deps/dependencies-client-spark-2.4 +index 72cd055c0..e9aeab399 100644 +--- a/dev/deps/dependencies-client-spark-2.4 ++++ b/dev/deps/dependencies-client-spark-2.4 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-spark-3.0 b/dev/deps/dependencies-client-spark-3.0 +index edd9f4264..67d0bfbe8 100644 +--- a/dev/deps/dependencies-client-spark-3.0 ++++ b/dev/deps/dependencies-client-spark-3.0 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-spark-3.1 b/dev/deps/dependencies-client-spark-3.1 +index 9372233f7..bf62abd24 100644 +--- a/dev/deps/dependencies-client-spark-3.1 ++++ b/dev/deps/dependencies-client-spark-3.1 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-spark-3.2 b/dev/deps/dependencies-client-spark-3.2 +index 3913a6fc1..2c04c88d4 100644 +--- a/dev/deps/dependencies-client-spark-3.2 ++++ b/dev/deps/dependencies-client-spark-3.2 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-spark-3.3 b/dev/deps/dependencies-client-spark-3.3 +index 945c35808..9218e8de7 100644 +--- a/dev/deps/dependencies-client-spark-3.3 ++++ b/dev/deps/dependencies-client-spark-3.3 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-spark-3.4 b/dev/deps/dependencies-client-spark-3.4 +index 6652a37cb..7af954c26 100644 +--- a/dev/deps/dependencies-client-spark-3.4 ++++ b/dev/deps/dependencies-client-spark-3.4 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-spark-3.5 b/dev/deps/dependencies-client-spark-3.5 +index a2cffa8a4..ff4997131 100644 +--- a/dev/deps/dependencies-client-spark-3.5 ++++ b/dev/deps/dependencies-client-spark-3.5 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-spark-4.0 b/dev/deps/dependencies-client-spark-4.0 +index 9c2fc4476..aa14aa42e 100644 +--- a/dev/deps/dependencies-client-spark-4.0 ++++ b/dev/deps/dependencies-client-spark-4.0 +@@ -18,7 +18,7 @@ + RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/dev/deps/dependencies-client-tez b/dev/deps/dependencies-client-tez +index 82dc2cb11..9f4b04a3f 100644 +--- a/dev/deps/dependencies-client-tez ++++ b/dev/deps/dependencies-client-tez +@@ -33,7 +33,7 @@ commons-configuration2/2.8.0//commons-configuration2-2.8.0.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar + commons-lang/2.6//commons-lang-2.6.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + commons-math3/3.1.1//commons-math3-3.1.1.jar + commons-net/3.9.0//commons-net-3.9.0.jar +diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server +index 9e8795b2d..f2ad0af93 100644 +--- a/dev/deps/dependencies-server ++++ b/dev/deps/dependencies-server +@@ -23,7 +23,7 @@ classgraph/4.8.138//classgraph-4.8.138.jar + commons-cli/1.5.0//commons-cli-1.5.0.jar + commons-crypto/1.0.0//commons-crypto-1.0.0.jar + commons-io/2.17.0//commons-io-2.17.0.jar +-commons-lang3/3.17.0//commons-lang3-3.17.0.jar ++commons-lang3/3.18.0//commons-lang3-3.18.0.jar + commons-logging/1.1.3//commons-logging-1.1.3.jar + failureaccess/1.0.2//failureaccess-1.0.2.jar + guava/33.1.0-jre//guava-33.1.0-jre.jar +diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala +index 191d74b22..e1a950917 100644 +--- a/project/CelebornBuild.scala ++++ b/project/CelebornBuild.scala +@@ -46,7 +46,7 @@ object Dependencies { + val commonsCryptoVersion = "1.0.0" + val commonsIoVersion = "2.17.0" + val commonsLoggingVersion = "1.1.3" +- val commonsLang3Version = "3.17.0" ++ val commonsLang3Version = "3.18.0" + val commonsCollectionsVersion = "3.2.2" + val findbugsVersion = "1.3.9" + val guavaVersion = "33.1.0-jre" +-- +2.50.0 \ No newline at end of file diff --git a/celeborn-0.6/pombump-deps.yaml b/celeborn-0.6/pombump-deps.yaml new file mode 100644 index 00000000000..3b84653ee0a --- /dev/null +++ b/celeborn-0.6/pombump-deps.yaml @@ -0,0 +1,10 @@ +patches: + - groupId: commons-beanutils + artifactId: commons-beanutils + version: 1.11.0 + - groupId: com.fasterxml.jackson.core + artifactId: jackson-core + version: 2.15.0 + - groupId: io.netty + artifactId: netty-codec-smtp + version: 4.1.128.Final diff --git a/celeborn-0.6/pombump-properties.yaml b/celeborn-0.6/pombump-properties.yaml new file mode 100644 index 00000000000..e20fe0788fc --- /dev/null +++ b/celeborn-0.6/pombump-properties.yaml @@ -0,0 +1,23 @@ +properties: + - property: commons-io.version + value: 2.14.0 + - property: commons-lang3.version + value: 3.18.0 + - property: hadoop.version + value: 3.4.2 + - property: jetty.version + value: 9.4.58.v20250814 + - property: maven.plugin.silencer.version + value: 1.7.19 + - property: netty.version + value: 4.1.125.Final + - property: protobuf.version + value: 3.25.5 + - property: ratis.version + value: 3.2.0 + - property: scala.binary.version + value: 2.13 + - property: scala.version + value: 2.13.17 + - property: spark.version + value: 3.5.5