diff --git a/dev-support/docker/Dockerfile_ubuntu_24_aarch64 b/dev-support/docker/Dockerfile_ubuntu_24_aarch64 new file mode 100644 index 0000000000000..9b44d78433897 --- /dev/null +++ b/dev-support/docker/Dockerfile_ubuntu_24_aarch64 @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM ubuntu:noble + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \ + && apt-get -q install -y --no-install-recommends python3 \ + && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ + && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \ + && apt-get -q update \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:noble) \ + && apt-get clean \ + && update-java-alternatives -s java-1.17.0-openjdk-arm64 \ + && rm -rf /var/lib/apt/lists/* + +RUN locale-gen en_US.UTF-8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +ENV PYTHONIOENCODING=utf-8 + +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME=/opt/maven +ENV PATH="${PATH}:${MAVEN_HOME}/bin" +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-arm64 + +####### +# Set env vars for SpotBugs 4.2.2 +####### +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Set env vars for Google Protobuf 3.21.12 +####### +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xms256m -Xmx3072m + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION true + +#### +# Install packages +#### +RUN pkg-resolver/install-maven.sh ubuntu:noble::arch64 +RUN pkg-resolver/install-spotbugs.sh ubuntu:noble::arch64 +RUN pkg-resolver/install-boost.sh ubuntu:noble::arch64 +RUN pkg-resolver/install-protobuf.sh ubuntu:noble::arch64 +RUN pkg-resolver/install-hadolint.sh ubuntu:noble::arch64 +RUN pkg-resolver/install-intel-isa-l.sh ubuntu:noble::arch64 + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index 364c8e2defe31..ef4d97ac0e5a5 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -6,6 +6,7 @@ "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", "ubuntu:noble": "ant", + "ubuntu:noble::arch64": "ant", "rockylinux:8": "ant" }, "apt-utils": { @@ -14,6 +15,7 @@ "debian:13": "apt-utils", "ubuntu:focal": "apt-utils", "ubuntu:noble": "apt-utils", + "ubuntu:noble::arch64": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, "automake": { @@ -22,6 +24,7 @@ "debian:13": "automake", "ubuntu:focal": "automake", "ubuntu:noble": "automake", + "ubuntu:noble::arch64": "automake", "ubuntu:focal::arch64": "automake", "rockylinux:8": "automake" }, @@ -31,6 +34,7 @@ "debian:13": "bats", "ubuntu:focal": "bats", "ubuntu:noble": "bats", + "ubuntu:noble::arch64": "bats", "ubuntu:focal::arch64": "bats", "rockylinux:8": "bats" }, @@ -40,6 +44,7 @@ "debian:13": "build-essential", "ubuntu:focal": "build-essential", "ubuntu:noble": "build-essential", + "ubuntu:noble::arch64": "build-essential", "ubuntu:focal::arch64": "build-essential" }, "bzip2": { @@ -63,6 +68,10 @@ "bzip2", "libbz2-dev" ], + "ubuntu:noble::arch64": [ + "bzip2", + "libbz2-dev" + ], "ubuntu:focal::arch64": [ "bzip2", "libbz2-dev" @@ -78,6 +87,7 @@ "debian:13": "clang", "ubuntu:focal": "clang", "ubuntu:noble": "clang", + "ubuntu:noble::arch64": "clang", "ubuntu:focal::arch64": "clang", "rockylinux:8": "clang" }, @@ -87,6 +97,7 @@ "debian:13": "cmake", "ubuntu:focal": "cmake", "ubuntu:noble": "cmake", + "ubuntu:noble::arch64": "cmake", "ubuntu:focal::arch64": "cmake" }, "curl": { @@ -110,6 +121,10 @@ "curl", "libcurl4-openssl-dev" ], + "ubuntu:noble::arch64": [ + "curl", + "libcurl4-openssl-dev" + ], "ubuntu:focal::arch64": [ "curl", "libcurl4-openssl-dev" @@ -125,6 +140,7 @@ "debian:13": "doxygen", "ubuntu:focal": "doxygen", "ubuntu:noble": "doxygen", + "ubuntu:noble::arch64": "doxygen", "ubuntu:focal::arch64": "doxygen" }, "dnf": { @@ -151,6 +167,10 @@ "fuse", "libfuse-dev" ], + "ubuntu:noble::arch64": [ + "fuse", + "libfuse-dev" + ], "ubuntu:focal::arch64": [ "fuse", "libfuse-dev" @@ -184,6 +204,10 @@ "gcc", "g++" ], + "ubuntu:noble::arch64": [ + "gcc", + "g++" + ], "ubuntu:focal::arch64": [ "gcc", "g++" @@ -195,6 +219,7 @@ "debian:13": "git", "ubuntu:focal": "git", "ubuntu:noble": "git", + "ubuntu:noble::arch64": "git", "ubuntu:focal::arch64": "git", "rockylinux:8": "git" }, @@ -204,6 +229,7 @@ "debian:13": "gnupg-agent", "ubuntu:focal": "gnupg-agent", "ubuntu:noble": "gnupg-agent", + "ubuntu:noble::arch64": "gnupg-agent", "ubuntu:focal::arch64": "gnupg-agent" }, "hugo": { @@ -212,6 +238,7 @@ "debian:13": "hugo", "ubuntu:focal": "hugo", "ubuntu:noble": "hugo", + "ubuntu:noble::arch64": "hugo", "ubuntu:focal::arch64": "hugo" }, "libbcprov-java": { @@ -220,6 +247,7 @@ "debian:13": "libbcprov-java", "ubuntu:focal": "libbcprov-java", "ubuntu:noble": "libbcprov-java", + "ubuntu:noble::arch64": "libbcprov-java", "ubuntu:focal::arch64": "libbcprov-java" }, "libtool": { @@ -228,6 +256,7 @@ "debian:13": "libtool", "ubuntu:focal": "libtool", "ubuntu:noble": "libtool", + "ubuntu:noble::arch64": "libtool", "ubuntu:focal::arch64": "libtool", "rockylinux:8": "libtool" }, @@ -237,6 +266,7 @@ "debian:13": "libssl-dev", "ubuntu:focal": "libssl-dev", "ubuntu:noble": "libssl-dev", + "ubuntu:noble::arch64": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", "rockylinux:8": "openssl-devel" }, @@ -267,6 +297,10 @@ "libprotobuf-dev", "libprotoc-dev" ], + "ubuntu:noble::arch64": [ + "libprotobuf-dev", + "libprotoc-dev" + ], "ubuntu:focal::arch64": [ "libprotobuf-dev", "libprotoc-dev" @@ -283,6 +317,7 @@ "debian:13": "libsasl2-dev", "ubuntu:focal": "libsasl2-dev", "ubuntu:noble": "libsasl2-dev", + "ubuntu:noble::arch64": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", "rockylinux:8": "cyrus-sasl-devel" }, @@ -292,6 +327,7 @@ "debian:13": "libsnappy-dev", "ubuntu:focal": "libsnappy-dev", "ubuntu:noble": "libsnappy-dev", + "ubuntu:noble::arch64": "libsnappy-dev", "ubuntu:focal::arch64": "libsnappy-dev" }, "zlib": { @@ -315,6 +351,10 @@ "libzstd-dev", "zlib1g-dev" ], + "ubuntu:noble::arch64": [ + "libzstd-dev", + "zlib1g-dev" + ], "ubuntu:focal::arch64": [ "libzstd-dev", "zlib1g-dev" @@ -330,13 +370,15 @@ "debian:13": "locales", "ubuntu:focal": "locales", "ubuntu:noble": "locales", + "ubuntu:noble::arch64": "locales", "ubuntu:focal::arch64": "locales" }, "libtirpc-devel": { "debian:12": "libtirpc-dev", "debian:13": "libtirpc-dev", "rockylinux:8": "libtirpc-devel", - "ubuntu:noble": "libtirpc-dev" + "ubuntu:noble": "libtirpc-dev", + "ubuntu:noble::arch64": "libtirpc-dev" }, "make": { "debian:11": "make", @@ -344,6 +386,7 @@ "debian:13": "make", "ubuntu:focal": "make", "ubuntu:noble": "make", + "ubuntu:noble::arch64": "make", "ubuntu:focal::arch64": "make", "rockylinux:8": "make" }, @@ -373,6 +416,12 @@ "openjdk-17-jdk", "openjdk-21-jdk" ], + "ubuntu:noble::arch64": [ + "temurin-25-jdk", + "openjdk-11-jdk", + "openjdk-17-jdk", + "openjdk-21-jdk" + ], "ubuntu:focal::arch64": [ "temurin-25-jdk", "openjdk-8-jdk", @@ -387,6 +436,7 @@ "debian:13": "pinentry-curses", "ubuntu:focal": "pinentry-curses", "ubuntu:noble": "pinentry-curses", + "ubuntu:noble::arch64": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", "rockylinux:8": "pinentry-curses" }, @@ -396,6 +446,7 @@ "debian:13": "pkg-config", "ubuntu:focal": "pkg-config", "ubuntu:noble": "pkg-config", + "ubuntu:noble::arch64": "pkg-config", "ubuntu:focal::arch64": "pkg-config", "rockylinux:8": "pkg-config" }, @@ -435,6 +486,13 @@ "python3-setuptools", "python3-wheel" ], + "ubuntu:noble::arch64": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], "ubuntu:focal::arch64": [ "python2.7", "python3", @@ -456,6 +514,7 @@ "debian:13": "rsync", "ubuntu:focal": "rsync", "ubuntu:noble": "rsync", + "ubuntu:noble::arch64": "rsync", "ubuntu:focal::arch64": "rsync", "rockylinux:8": "rsync" }, @@ -465,6 +524,7 @@ "debian:13": "shellcheck", "ubuntu:focal": "shellcheck", "ubuntu:noble": "shellcheck", + "ubuntu:noble::arch64": "shellcheck", "ubuntu:focal::arch64": "shellcheck" }, "shasum": { @@ -474,6 +534,7 @@ "debian:11": "software-properties-common", "ubuntu:focal": "software-properties-common", "ubuntu:noble": "software-properties-common", + "ubuntu:noble::arch64": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" }, "sudo": { @@ -482,6 +543,7 @@ "debian:13": "sudo", "ubuntu:focal": "sudo", "ubuntu:noble": "sudo", + "ubuntu:noble::arch64": "sudo", "ubuntu:focal::arch64": "sudo", "rockylinux:8": "sudo" }, @@ -491,6 +553,7 @@ "debian:13": "valgrind", "ubuntu:focal": "valgrind", "ubuntu:noble": "valgrind", + "ubuntu:noble::arch64": "valgrind", "ubuntu:focal::arch64": "valgrind", "rockylinux:8": "valgrind" }, @@ -500,6 +563,7 @@ "debian:13": "yasm", "ubuntu:focal": "yasm", "ubuntu:noble": "yasm", + "ubuntu:noble::arch64": "yasm", "ubuntu:focal::arch64": "yasm" } } diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index 3f556de5ad5b5..06ddfabcd6acc 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -2,6 +2,7 @@ "ubuntu:focal", "ubuntu:focal::arch64", "ubuntu:noble", + "ubuntu:noble::arch64", "rockylinux:8", "debian:11", "debian:12",