Closed
Conversation
apache/spark PR for 3.5.3 is here apache/spark-docker#72
Diff for dbdc358:diff --git a/_bashbrew-cat b/_bashbrew-cat
index dfcdb83..5904a1b 100644
--- a/_bashbrew-cat
+++ b/_bashbrew-cat
@@ -21,45 +21,45 @@ Architectures: amd64, arm64v8
GitCommit: 6f10acb452a360966ec9a971641280b7b4a3619a
Directory: 3.4.3/scala2.12-java11-ubuntu
-Tags: 3.5.2-scala2.12-java11-python3-r-ubuntu
+Tags: 3.5.3-scala2.12-java11-python3-r-ubuntu
Architectures: amd64, arm64v8
-GitCommit: b9f1f8e8ebed1959c2be3864a114b52f67519092
-Directory: 3.5.2/scala2.12-java11-python3-r-ubuntu
+GitCommit: e498bfc1509009c74151b0df559867c73be7deb9
+Directory: 3.5.3/scala2.12-java11-python3-r-ubuntu
-Tags: 3.5.2-scala2.12-java11-python3-ubuntu, 3.5.2-python3, 3.5.2, python3, latest
+Tags: 3.5.3-scala2.12-java11-python3-ubuntu, 3.5.3-python3, 3.5.3, python3, latest
Architectures: amd64, arm64v8
-GitCommit: b9f1f8e8ebed1959c2be3864a114b52f67519092
-Directory: 3.5.2/scala2.12-java11-python3-ubuntu
+GitCommit: e498bfc1509009c74151b0df559867c73be7deb9
+Directory: 3.5.3/scala2.12-java11-python3-ubuntu
-Tags: 3.5.2-scala2.12-java11-r-ubuntu, 3.5.2-r, r
+Tags: 3.5.3-scala2.12-java11-r-ubuntu, 3.5.3-r, r
Architectures: amd64, arm64v8
-GitCommit: b9f1f8e8ebed1959c2be3864a114b52f67519092
-Directory: 3.5.2/scala2.12-java11-r-ubuntu
+GitCommit: e498bfc1509009c74151b0df559867c73be7deb9
+Directory: 3.5.3/scala2.12-java11-r-ubuntu
-Tags: 3.5.2-scala2.12-java11-ubuntu, 3.5.2-scala, scala
+Tags: 3.5.3-scala2.12-java11-ubuntu, 3.5.3-scala, scala
Architectures: amd64, arm64v8
-GitCommit: b9f1f8e8ebed1959c2be3864a114b52f67519092
-Directory: 3.5.2/scala2.12-java11-ubuntu
+GitCommit: e498bfc1509009c74151b0df559867c73be7deb9
+Directory: 3.5.3/scala2.12-java11-ubuntu
-Tags: 3.5.2-scala2.12-java17-python3-r-ubuntu
+Tags: 3.5.3-scala2.12-java17-python3-r-ubuntu
Architectures: amd64, arm64v8
-GitCommit: b9f1f8e8ebed1959c2be3864a114b52f67519092
-Directory: 3.5.2/scala2.12-java17-python3-r-ubuntu
+GitCommit: e498bfc1509009c74151b0df559867c73be7deb9
+Directory: 3.5.3/scala2.12-java17-python3-r-ubuntu
-Tags: 3.5.2-scala2.12-java17-python3-ubuntu, 3.5.2-java17-python3, 3.5.2-java17, python3-java17
+Tags: 3.5.3-scala2.12-java17-python3-ubuntu, 3.5.3-java17-python3, 3.5.3-java17, python3-java17
Architectures: amd64, arm64v8
-GitCommit: b9f1f8e8ebed1959c2be3864a114b52f67519092
-Directory: 3.5.2/scala2.12-java17-python3-ubuntu
+GitCommit: e498bfc1509009c74151b0df559867c73be7deb9
+Directory: 3.5.3/scala2.12-java17-python3-ubuntu
-Tags: 3.5.2-scala2.12-java17-r-ubuntu, 3.5.2-java17-r
+Tags: 3.5.3-scala2.12-java17-r-ubuntu, 3.5.3-java17-r
Architectures: amd64, arm64v8
-GitCommit: b9f1f8e8ebed1959c2be3864a114b52f67519092
-Directory: 3.5.2/scala2.12-java17-r-ubuntu
+GitCommit: e498bfc1509009c74151b0df559867c73be7deb9
+Directory: 3.5.3/scala2.12-java17-r-ubuntu
-Tags: 3.5.2-scala2.12-java17-ubuntu, 3.5.2-java17-scala
+Tags: 3.5.3-scala2.12-java17-ubuntu, 3.5.3-java17-scala
Architectures: amd64, arm64v8
-GitCommit: b9f1f8e8ebed1959c2be3864a114b52f67519092
-Directory: 3.5.2/scala2.12-java17-ubuntu
+GitCommit: e498bfc1509009c74151b0df559867c73be7deb9
+Directory: 3.5.3/scala2.12-java17-ubuntu
Tags: 4.0.0-preview2-scala2.13-java17-python3-r-ubuntu
Architectures: amd64, arm64v8
diff --git a/_bashbrew-list b/_bashbrew-list
index 5a0fa41..61f802e 100644
--- a/_bashbrew-list
+++ b/_bashbrew-list
@@ -6,22 +6,22 @@ spark:3.4.3-scala2.12-java11-python3-r-ubuntu
spark:3.4.3-scala2.12-java11-python3-ubuntu
spark:3.4.3-scala2.12-java11-r-ubuntu
spark:3.4.3-scala2.12-java11-ubuntu
-spark:3.5.2
-spark:3.5.2-java17
-spark:3.5.2-java17-python3
-spark:3.5.2-java17-r
-spark:3.5.2-java17-scala
-spark:3.5.2-python3
-spark:3.5.2-r
-spark:3.5.2-scala
-spark:3.5.2-scala2.12-java11-python3-r-ubuntu
-spark:3.5.2-scala2.12-java11-python3-ubuntu
-spark:3.5.2-scala2.12-java11-r-ubuntu
-spark:3.5.2-scala2.12-java11-ubuntu
-spark:3.5.2-scala2.12-java17-python3-r-ubuntu
-spark:3.5.2-scala2.12-java17-python3-ubuntu
-spark:3.5.2-scala2.12-java17-r-ubuntu
-spark:3.5.2-scala2.12-java17-ubuntu
+spark:3.5.3
+spark:3.5.3-java17
+spark:3.5.3-java17-python3
+spark:3.5.3-java17-r
+spark:3.5.3-java17-scala
+spark:3.5.3-python3
+spark:3.5.3-r
+spark:3.5.3-scala
+spark:3.5.3-scala2.12-java11-python3-r-ubuntu
+spark:3.5.3-scala2.12-java11-python3-ubuntu
+spark:3.5.3-scala2.12-java11-r-ubuntu
+spark:3.5.3-scala2.12-java11-ubuntu
+spark:3.5.3-scala2.12-java17-python3-r-ubuntu
+spark:3.5.3-scala2.12-java17-python3-ubuntu
+spark:3.5.3-scala2.12-java17-r-ubuntu
+spark:3.5.3-scala2.12-java17-ubuntu
spark:4.0.0-preview2
spark:4.0.0-preview2-java21
spark:4.0.0-preview2-java21-python3
diff --git a/_bashbrew-list-build-order b/_bashbrew-list-build-order
index 3536513..89ac45d 100644
--- a/_bashbrew-list-build-order
+++ b/_bashbrew-list-build-order
@@ -1,7 +1,7 @@
spark:3.4.3-scala
spark:3.4.3-scala2.12-java11-python3-r-ubuntu
-spark:3.5.2-java17-scala
-spark:3.5.2-scala2.12-java17-python3-r-ubuntu
+spark:3.5.3-java17-scala
+spark:3.5.3-scala2.12-java17-python3-r-ubuntu
spark:4.0.0-preview2-java21-scala
spark:4.0.0-preview2-scala
spark:4.0.0-preview2-scala2.13-java17-python3-r-ubuntu
@@ -10,8 +10,8 @@ spark:python3-java17
spark:scala
spark:3.4.3
spark:3.4.3-r
-spark:3.5.2-java17-r
-spark:3.5.2-scala2.12-java11-python3-r-ubuntu
+spark:3.5.3-java17-r
+spark:3.5.3-scala2.12-java11-python3-r-ubuntu
spark:4.0.0-preview2
spark:4.0.0-preview2-java21
spark:4.0.0-preview2-java21-r
diff --git a/spark_3.5.2-java17-r/Dockerfile b/spark_3.5.3-java17-r/Dockerfile
similarity index 95%
rename from spark_3.5.2-java17-r/Dockerfile
rename to spark_3.5.3-java17-r/Dockerfile
index 46c55fc..171fa66 100644
--- a/spark_3.5.2-java17-r/Dockerfile
+++ b/spark_3.5.3-java17-r/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.5.2-scala2.12-java17-ubuntu
+FROM spark:3.5.3-scala2.12-java17-ubuntu
USER root
diff --git a/spark_3.5.2-java17-scala/Dockerfile b/spark_3.5.3-java17-scala/Dockerfile
similarity index 89%
rename from spark_3.5.2-java17-scala/Dockerfile
rename to spark_3.5.3-java17-scala/Dockerfile
index 280bd0c..5f49c91 100644
--- a/spark_3.5.2-java17-scala/Dockerfile
+++ b/spark_3.5.3-java17-scala/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM eclipse-temurin:17-jre-jammy
+FROM eclipse-temurin:17-jammy
ARG spark_uid=185
@@ -36,9 +36,9 @@ RUN set -ex; \
# Install Apache Spark
# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz.asc \
- GPG_KEY=D76E23B9F11B5BF6864613C4F7051850A0AF904D
+ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz \
+ SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz.asc \
+ GPG_KEY=0A2D660358B6F6F8071FD16F6606986CF5A8447C
RUN set -ex; \
export SPARK_TMP="$(mktemp -d)"; \
@@ -55,10 +55,12 @@ RUN set -ex; \
tar -xf spark.tgz --strip-components=1; \
chown -R spark:spark .; \
mv jars /opt/spark/; \
+ mv RELEASE /opt/spark/; \
mv bin /opt/spark/; \
mv sbin /opt/spark/; \
mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
mv examples /opt/spark/; \
+ ln -s "$(basename $(ls /opt/spark/examples/jars/spark-examples_*.jar))" /opt/spark/examples/jars/spark-examples.jar; \
mv kubernetes/tests /opt/spark/; \
mv data /opt/spark/; \
mv python/pyspark /opt/spark/python/pyspark/; \
diff --git a/spark_3.5.2-java17-scala/entrypoint.sh b/spark_3.5.3-java17-scala/entrypoint.sh
similarity index 100%
rename from spark_3.5.2-java17-scala/entrypoint.sh
rename to spark_3.5.3-java17-scala/entrypoint.sh
diff --git a/spark_3.5.2-scala2.12-java17-python3-r-ubuntu/Dockerfile b/spark_3.5.3-scala2.12-java11-python3-r-ubuntu/Dockerfile
similarity index 96%
rename from spark_3.5.2-scala2.12-java17-python3-r-ubuntu/Dockerfile
rename to spark_3.5.3-scala2.12-java11-python3-r-ubuntu/Dockerfile
index 80dda3b..b7f4cc6 100644
--- a/spark_3.5.2-scala2.12-java17-python3-r-ubuntu/Dockerfile
+++ b/spark_3.5.3-scala2.12-java11-python3-r-ubuntu/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.5.2-scala2.12-java17-ubuntu
+FROM spark:3.5.3-scala2.12-java11-ubuntu
USER root
diff --git a/spark_3.5.2-scala2.12-java11-python3-r-ubuntu/Dockerfile b/spark_3.5.3-scala2.12-java17-python3-r-ubuntu/Dockerfile
similarity index 96%
rename from spark_3.5.2-scala2.12-java11-python3-r-ubuntu/Dockerfile
rename to spark_3.5.3-scala2.12-java17-python3-r-ubuntu/Dockerfile
index f29cf36..b65c879 100644
--- a/spark_3.5.2-scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ b/spark_3.5.3-scala2.12-java17-python3-r-ubuntu/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.5.2-scala2.12-java11-ubuntu
+FROM spark:3.5.3-scala2.12-java17-ubuntu
USER root
diff --git a/spark_latest/Dockerfile b/spark_latest/Dockerfile
index b75bf37..60b10b9 100644
--- a/spark_latest/Dockerfile
+++ b/spark_latest/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.5.2-scala2.12-java11-ubuntu
+FROM spark:3.5.3-scala2.12-java11-ubuntu
USER root
diff --git a/spark_python3-java17/Dockerfile b/spark_python3-java17/Dockerfile
index 33bd4cd..f0e7d87 100644
--- a/spark_python3-java17/Dockerfile
+++ b/spark_python3-java17/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.5.2-scala2.12-java17-ubuntu
+FROM spark:3.5.3-scala2.12-java17-ubuntu
USER root
diff --git a/spark_r/Dockerfile b/spark_r/Dockerfile
index ef88397..3b76da8 100644
--- a/spark_r/Dockerfile
+++ b/spark_r/Dockerfile
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM spark:3.5.2-scala2.12-java11-ubuntu
+FROM spark:3.5.3-scala2.12-java11-ubuntu
USER root
diff --git a/spark_scala/Dockerfile b/spark_scala/Dockerfile
index e378d88..0a4360a 100644
--- a/spark_scala/Dockerfile
+++ b/spark_scala/Dockerfile
@@ -36,9 +36,9 @@ RUN set -ex; \
# Install Apache Spark
# https://downloads.apache.org/spark/KEYS
-ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz \
- SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz.asc \
- GPG_KEY=D76E23B9F11B5BF6864613C4F7051850A0AF904D
+ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz \
+ SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz.asc \
+ GPG_KEY=0A2D660358B6F6F8071FD16F6606986CF5A8447C
RUN set -ex; \
export SPARK_TMP="$(mktemp -d)"; \
@@ -55,10 +55,12 @@ RUN set -ex; \
tar -xf spark.tgz --strip-components=1; \
chown -R spark:spark .; \
mv jars /opt/spark/; \
+ mv RELEASE /opt/spark/; \
mv bin /opt/spark/; \
mv sbin /opt/spark/; \
mv kubernetes/dockerfiles/spark/decom.sh /opt/; \
mv examples /opt/spark/; \
+ ln -s "$(basename $(ls /opt/spark/examples/jars/spark-examples_*.jar))" /opt/spark/examples/jars/spark-examples.jar; \
mv kubernetes/tests /opt/spark/; \
mv data /opt/spark/; \
mv python/pyspark /opt/spark/python/pyspark/; \Relevant Maintainers:
|
Member
Author
|
Closing in favor of #17698 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
apache/spark PR for 3.5.3 is here apache/spark-docker#72
@itholic