Skip to content

Commit 0c11090

Browse files
octo-sts[bot]MickeyPvX
octo-sts[bot]
and
MickeyPvX
authored
Adding VersionStream for celeborn-0.6 (#69794)
Signed-off-by: Michael Paul <[email protected]> Co-authored-by: octo-sts[bot] <[email protected]> Co-authored-by: Michael Paul <[email protected]>
1 parent 943b50e commit 0c11090

File tree

4 files changed

+463
-0
lines changed

4 files changed

+463
-0
lines changed

‎celeborn-0.6.yaml‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
package:
2+
name: celeborn-0.6
3+
version: "0.6.1"
4+
epoch: 0 # GHSA-jq43-27x9-3v86
5+
description: "Apache Celeborn - A Remote Shuffle Service for Distributed Data Processing Engines"
6+
copyright:
7+
- license: Apache-2.0
8+
dependencies:
9+
provides:
10+
- celeborn=${{package.full-version}}
11+
runtime:
12+
- bash
13+
- coreutils
14+
- glibc-locale-en
15+
- libjemalloc2
16+
- openjdk-${{vars.java-version}}-default-jvm
17+
- openssh
18+
- procps
19+
- sed
20+
- tzdata
21+
22+
vars:
23+
base-package-name: celeborn
24+
java-version: 8
25+
26+
environment:
27+
contents:
28+
packages:
29+
- bash
30+
- busybox
31+
- git
32+
- libstdc++
33+
- maven
34+
- openjdk-${{vars.java-version}}-default-jdk
35+
environment:
36+
JAVA_HOME: /usr/lib/jvm/java-1.${{vars.java-version}}-openjdk
37+
LANG: en_US.UTF-8
38+
39+
pipeline:
40+
- uses: git-checkout
41+
with:
42+
expected-commit: 29d03860c0f219eec68854e2c8535b14c899fdfa
43+
repository: https://github.com/apache/celeborn.git
44+
tag: v${{package.version}}
45+
46+
- uses: maven/pombump
47+
48+
- uses: patch
49+
with:
50+
patches: commons-lang3-3.18.0.patch
51+
52+
- name: Build / install Celeborn
53+
runs: |
54+
# Build parts taken from https://github.com/apache/celeborn/blob/main/build/make-distribution.sh
55+
56+
dest=${{targets.contextdir}}/usr/share/java/${{vars.base-package-name}}
57+
mkdir -p \
58+
$dest \
59+
$dest/bin \
60+
$dest/conf \
61+
$dest/jars \
62+
$dest/sbin
63+
64+
SCALA_VERSION="$(mvn help:evaluate -Dexpression=scala.binary.version 2>/dev/null \
65+
| grep -v "INFO" \
66+
| grep -v "WARNING" \
67+
| tail -n 1)"
68+
VERSION="$(mvn help:evaluate -Dexpression=project.version 2>/dev/null \
69+
| grep -v "INFO" \
70+
| grep -v "WARNING" \
71+
| tail -n 1)"
72+
73+
echo "Celeborn ${{package.version}} (git revision $(git rev-parse --short HEAD 2>/dev/null))" > $dest/RELEASE
74+
75+
mvn clean package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip -pl master,worker -am
76+
77+
for dir in master worker
78+
do
79+
mkdir -p "$dest/$dir-jars"
80+
81+
cp "$dir"/target/celeborn-"$dir"_"$SCALA_VERSION"-"$VERSION".jar "$dest/$dir-jars/"
82+
cp "$dir"/target/scala-"$SCALA_VERSION"/jars/*.jar "$dest/jars/"
83+
84+
for jar in "$dir/target/scala-$SCALA_VERSION/jars"/*;
85+
do
86+
(cd "$dest/$dir-jars"; ln -snf "../jars/$(basename $jar)" .)
87+
done
88+
done
89+
90+
cp conf/*.template $dest/conf
91+
cp -r bin $dest
92+
cp -r sbin $dest
93+
94+
subpackages:
95+
- name: ${{package.name}}-compat
96+
description: Compat package for ${{package.name}}
97+
dependencies:
98+
provides:
99+
- ${{vars.base-package-name}}-compat=${{package.full-version}}
100+
pipeline:
101+
- runs: |
102+
mkdir -p ${{targets.contextdir}}/opt/${{vars.base-package-name}}
103+
104+
for dir in bin conf jars master-jars sbin worker-jars
105+
do
106+
ln -sf /usr/share/java/${{vars.base-package-name}}/$dir ${{targets.contextdir}}/opt/${{vars.base-package-name}}/$dir
107+
done
108+
test:
109+
environment:
110+
contents:
111+
packages:
112+
- ${{package.name}}
113+
pipeline:
114+
- name: Test symlinks
115+
runs: |
116+
for dir in bin conf jars master-jars sbin worker-jars
117+
do
118+
stat -L /opt/${{vars.base-package-name}}/$dir
119+
done
120+
121+
test:
122+
environment:
123+
contents:
124+
packages:
125+
- ${{package.name}}-compat
126+
- wait-for-it
127+
# Some of these are set to match defaults in the external counterpart image
128+
environment:
129+
CELEBORN_HOME: /opt/celeborn
130+
CELEBORN_JEMALLOC_PATH: /usr/lib/libjemalloc.so.2
131+
CELEBORN_PREFER_JEMALLOC: true
132+
JAVA_HOME: /usr/lib/jvm/java-1.${{vars.java-version}}-openjdk
133+
LC_ALL: en_US.UTF-8
134+
pipeline:
135+
- name: Test starting master
136+
runs: |
137+
cd $CELEBORN_HOME/conf
138+
cp log4j2.xml.template log4j2.xml
139+
140+
cat >> celeborn-defaults.conf <<EOF
141+
celeborn.master.host 127.0.0.1
142+
celeborn.master.port 9097
143+
celeborn.network.bind.preferIpAddress true
144+
celeborn.worker.storage.dirs=$CELEBORN_HOME/shuffle
145+
EOF
146+
147+
cd $CELEBORN_HOME
148+
./sbin/start-master.sh
149+
150+
wait-for-it 127.0.0.1:9097
151+
152+
timeout 10 tail -f $CELEBORN_HOME/logs/celeborn* | grep "Master: Master started."
153+
154+
# Get master IP
155+
ip="$(grep "Starting RPC Server \[Master\] on" $CELEBORN_HOME/logs/celeborn* | grep -o -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+' | head -1)"
156+
157+
./sbin/start-worker.sh celeborn://$ip
158+
159+
timeout 10 tail -f $CELEBORN_HOME/logs/celeborn* | grep "Worker: Worker started."
160+
161+
update:
162+
enabled: true
163+
github:
164+
identifier: apache/celeborn
165+
tag-filter: v0.6.
166+
strip-prefix: v
167+
ignore-regex-patterns:
168+
- -rc[0-9]+$

0 commit comments

Comments
 (0)