diff --git a/BUILDING.txt b/BUILDING.txt index 1e2a1fef1a098e..f0c11cfc30eb9a 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -29,7 +29,11 @@ This requires a recent version of docker (1.4.1 and higher are known to work). On Linux / Mac: Install Docker and run this command: - $ ./start-build-env.sh + $ ./start-build-env.sh [OS platform] + + - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24, windows_10]. + Default is 'ubuntu_20'. + Note: Currently only default ('ubuntu_20') is supported on arm machine The prompt which is then presented is located at a mounted version of the source tree and all required tools for testing and building have been installed and configured. @@ -457,7 +461,7 @@ path. This is necessary at least for Homebrewed OpenSSL. ---------------------------------------------------------------------------------- -Building on CentOS 8 +Building on Rocky Linux 8 ---------------------------------------------------------------------------------- @@ -478,7 +482,7 @@ Building on CentOS 8 $ sudo make install $ cd .. -* Install libraries provided by CentOS 8. +* Install libraries provided by Rocky Linux 8. $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel * Install GCC 9.3.0 diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 51225268b653aa..672061ea09ad92 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -83,24 +83,18 @@ pipeline { // optional stages after the first one. stage ('setup sources') { steps { - dir("${WORKSPACE}/centos-7") { - sh '''#!/usr/bin/env bash - - cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7 - ''' - } - dir("${WORKSPACE}/centos-8") { + dir("${WORKSPACE}/rockylinux-8") { sh '''#!/usr/bin/env bash - cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8 + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8 ''' } - dir("${WORKSPACE}/debian-10") { + dir("${WORKSPACE}/debian-11") { sh '''#!/usr/bin/env bash - cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10 + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-11 ''' } @@ -117,58 +111,12 @@ pipeline { // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Centos 7. - stage ('precommit-run Centos 7') { - environment { - SOURCEDIR = "${WORKSPACE}/centos-7/src" - PATCHDIR = "${WORKSPACE}/centos-7/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7" - IS_OPTIONAL = 1 - } - - steps { - withCredentials(getGithubCreds()) { - sh '''#!/usr/bin/env bash - - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" run_ci - ''' - } - } - - post { - // Since this is an optional platform, we want to copy the artifacts - // and archive it only if the build fails, to help with debugging. - failure { - sh '''#!/usr/bin/env bash - - cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}" - ''' - archiveArtifacts "out/**" - } - - cleanup() { - script { - sh '''#!/usr/bin/env bash - - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc - ''' - } - } - } - } - - // This is an optional stage which runs only when there's a change in - // C++/C++ build/platform. - // This stage serves as a means of cross platform validation, which is - // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Centos 8. - stage ('precommit-run Centos 8') { + // break the Hadoop build on Rocky Linux 8. + stage ('precommit-run Rocky Linux 8') { environment { - SOURCEDIR = "${WORKSPACE}/centos-8/src" - PATCHDIR = "${WORKSPACE}/centos-8/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8" + SOURCEDIR = "${WORKSPACE}/rockylinux-8/src" + PATCHDIR = "${WORKSPACE}/rockylinux-8/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_rockylinux_8" IS_OPTIONAL = 1 } @@ -188,7 +136,7 @@ pipeline { failure { sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}" + cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}" ''' archiveArtifacts "out/**" } @@ -209,12 +157,12 @@ pipeline { // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Debian 10. - stage ('precommit-run Debian 10') { + // break the Hadoop build on Debian 11. + stage ('precommit-run Debian 11') { environment { - SOURCEDIR = "${WORKSPACE}/debian-10/src" - PATCHDIR = "${WORKSPACE}/debian-10/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" + SOURCEDIR = "${WORKSPACE}/debian-11/src" + PATCHDIR = "${WORKSPACE}/debian-11/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11" IS_OPTIONAL = 1 } @@ -234,7 +182,7 @@ pipeline { failure { sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}" + cp -Rp "${WORKSPACE}/debian-11/out" "${WORKSPACE}" ''' archiveArtifacts "out/**" } diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 3b71e622a575ef..6beef860a9b39e 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -44,9 +44,14 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ ###### # hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \ && apt-get -q install -y --no-install-recommends python3 \ + && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ + && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \ + && apt-get -q update \ && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ && apt-get clean \ + && update-java-alternatives -s java-1.8.0-openjdk-amd64 \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 26ad10edd6f238..b9249f5aec9d38 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -44,10 +44,15 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ ###### # hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \ && apt-get -q install -y --no-install-recommends python3 \ && apt-get -q install -y --no-install-recommends phantomjs \ + && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ + && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \ + && apt-get -q update \ && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \ && apt-get clean \ + && update-java-alternatives -s java-1.8.0-openjdk-arm64 \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 deleted file mode 100644 index b97e59969a7600..00000000000000 --- a/dev-support/docker/Dockerfile_centos_7 +++ /dev/null @@ -1,108 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Dockerfile for installing the necessary dependencies for building Hadoop. -# See BUILDING.txt. - -FROM centos:7 - -WORKDIR /root - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -###### -# Platform package dependency resolver -###### -COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ - && chmod a+r pkg-resolver/*.json - -###### -# Centos 7 has reached its EOL and the packages -# are no longer available on mirror.centos.org site. -# Please see https://www.centos.org/centos-linux-eol/ -###### -RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7 - -###### -# Install packages from yum -###### -# hadolint ignore=DL3008,SC2046 -RUN yum update -y \ - && yum groupinstall -y "Development Tools" \ - && yum install -y \ - centos-release-scl \ - python3 - -# Apply the script again because centos-release-scl creates new YUM repo files -RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7 - -# hadolint ignore=DL3008,SC2046 -RUN yum install -y $(pkg-resolver/resolve.py centos:7) - -# Set GCC 9 as the default C/C++ compiler -RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc -SHELL ["/bin/bash", "--login", "-c"] - -###### -# Set the environment variables needed for CMake -# to find and use GCC 9 for compilation -###### -ENV GCC_HOME "/opt/rh/devtoolset-9" -ENV CC "${GCC_HOME}/root/usr/bin/gcc" -ENV CXX "${GCC_HOME}/root/usr/bin/g++" -ENV SHLVL 1 -ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" -ENV PCP_DIR "${GCC_HOME}/root" -ENV MANPATH "${GCC_HOME}/root/usr/share/man:" -ENV PATH "${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig" -ENV INFOPATH "${GCC_HOME}/root/usr/share/info" - -# TODO: Set locale - -###### -# Set env vars required to build Hadoop -###### -ENV MAVEN_HOME /opt/maven -ENV PATH "${PATH}:${MAVEN_HOME}/bin" -# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 - -####### -# Set env vars for SpotBugs -####### -ENV SPOTBUGS_HOME /opt/spotbugs - -####### -# Set env vars for Google Protobuf 3.21.12 -####### -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" - -###### -# Install packages -###### -RUN pkg-resolver/install-maven.sh centos:7 -RUN pkg-resolver/install-cmake.sh centos:7 -RUN pkg-resolver/install-zstandard.sh centos:7 -RUN pkg-resolver/install-yasm.sh centos:7 -RUN pkg-resolver/install-protobuf.sh centos:7 -RUN pkg-resolver/install-boost.sh centos:7 -RUN pkg-resolver/install-spotbugs.sh centos:7 -RUN pkg-resolver/install-nodejs.sh centos:7 -RUN pkg-resolver/install-git.sh centos:7 -RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_11 similarity index 71% rename from dev-support/docker/Dockerfile_debian_10 rename to dev-support/docker/Dockerfile_debian_11 index 71446b27f686ba..2dc47eee6ac6ef 100644 --- a/dev-support/docker/Dockerfile_debian_10 +++ b/dev-support/docker/Dockerfile_debian_11 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM debian:10 +FROM debian:11 WORKDIR /root @@ -29,26 +29,24 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras -ENV DEBIAN_FRONTEND noninteractive -ENV DEBCONF_TERSE true +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_TERSE=true ###### # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/install-pkg-resolver.sh -RUN pkg-resolver/install-pkg-resolver.sh debian:10 +RUN pkg-resolver/install-pkg-resolver.sh debian:11 ###### # Install packages from apt ###### # hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \ - && echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \ - && apt-get -q update \ - && apt-get -q install -y --no-install-recommends -t bullseye $(pkg-resolver/resolve.py --release=bullseye debian:10) \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:11) \ && apt-get clean \ + && update-java-alternatives -s java-1.11.0-openjdk-amd64 \ && rm -rf /var/lib/apt/lists/* # TODO : Set locale @@ -56,38 +54,38 @@ RUN apt-get -q update \ ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /usr +ENV MAVEN_HOME=/usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 ####### # Set env vars for SpotBugs 4.2.2 ####### -ENV SPOTBUGS_HOME /opt/spotbugs +ENV SPOTBUGS_HOME=/opt/spotbugs ####### # Set env vars for Google Protobuf 3.21.12 ####### -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" +ENV PROTOBUF_HOME=/opt/protobuf +ENV PATH="${PATH}:/opt/protobuf/bin" ### # Avoid out of memory errors in builds ### -ENV MAVEN_OPTS -Xms256m -Xmx3072m +ENV MAVEN_OPTS="-Xms256m -Xmx3072m" # Skip gpg verification when downloading Yetus via yetus-wrapper -ENV HADOOP_SKIP_YETUS_VERIFICATION true +ENV HADOOP_SKIP_YETUS_VERIFICATION=true #### # Install packages #### -RUN pkg-resolver/install-cmake.sh debian:10 -RUN pkg-resolver/install-spotbugs.sh debian:10 -RUN pkg-resolver/install-boost.sh debian:10 -RUN pkg-resolver/install-protobuf.sh debian:10 -RUN pkg-resolver/install-hadolint.sh debian:10 -RUN pkg-resolver/install-intel-isa-l.sh debian:10 +RUN pkg-resolver/install-cmake.sh debian:11 +RUN pkg-resolver/install-spotbugs.sh debian:11 +RUN pkg-resolver/install-boost.sh debian:11 +RUN pkg-resolver/install-protobuf.sh debian:11 +RUN pkg-resolver/install-hadolint.sh debian:11 +RUN pkg-resolver/install-intel-isa-l.sh debian:11 ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_rockylinux_8 similarity index 55% rename from dev-support/docker/Dockerfile_centos_8 rename to dev-support/docker/Dockerfile_rockylinux_8 index ee0c8e88f74e49..63c22c5e3e1f97 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_rockylinux_8 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM centos:8 +FROM rockylinux:8 WORKDIR /root @@ -30,25 +30,18 @@ COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ && chmod a+r pkg-resolver/*.json -###### -# Centos 8 has reached its EOL and the packages -# are no longer available on mirror.centos.org site. -# Please see https://www.centos.org/centos-linux-eol/ -###### -RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8 - ###### # Install packages from yum ###### # hadolint ignore=DL3008,SC2046 RUN yum update -y \ && yum install -y python3 \ - && yum install -y $(pkg-resolver/resolve.py centos:8) + && yum install -y $(pkg-resolver/resolve.py rockylinux:8) #### # Install EPEL #### -RUN pkg-resolver/install-epel.sh centos:8 +RUN pkg-resolver/install-epel.sh rockylinux:8 RUN dnf --enablerepo=powertools install -y \ doxygen \ @@ -69,50 +62,65 @@ SHELL ["/bin/bash", "--login", "-c"] # Set the environment variables needed for CMake # to find and use GCC 9 for compilation ###### -ENV GCC_HOME "/opt/rh/gcc-toolset-9" -ENV CC "${GCC_HOME}/root/usr/bin/gcc" -ENV CXX "${GCC_HOME}/root/usr/bin/g++" -ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD" -ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl" -ENV SHLVL 1 -ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles" -ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1" -ENV MODULESHOME "/usr/share/Modules" -ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" -ENV PCP_DIR "${GCC_HOME}/root" -ENV MANPATH "${GCC_HOME}/root/usr/share/man::" -ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig" -ENV INFOPATH "${GCC_HOME}/root/usr/share/info" +ENV GCC_HOME="/opt/rh/gcc-toolset-9" +ENV CC="${GCC_HOME}/root/usr/bin/gcc" +ENV CXX="${GCC_HOME}/root/usr/bin/g++" +ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD" +ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl" +ENV SHLVL=1 +ENV MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles" +ENV MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1" +ENV MODULESHOME="/usr/share/Modules" +ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" +ENV PCP_DIR="${GCC_HOME}/root" +ENV MANPATH="${GCC_HOME}/root/usr/share/man::" +ENV PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig" +ENV INFOPATH="${GCC_HOME}/root/usr/share/info" # TODO: Set locale ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /opt/maven -ENV PATH "${PATH}:${MAVEN_HOME}/bin" +ENV MAVEN_HOME=/opt/maven +ENV PATH="${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 +ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 ####### # Set env vars for SpotBugs ####### -ENV SPOTBUGS_HOME /opt/spotbugs +ENV SPOTBUGS_HOME=/opt/spotbugs ####### # Set env vars for Google Protobuf 3.21.12 ####### -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" +ENV PROTOBUF_HOME=/opt/protobuf +ENV PATH="${PATH}:${PROTOBUF_HOME}/bin" + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION=true ###### # Install packages ###### -RUN pkg-resolver/install-maven.sh centos:8 -RUN pkg-resolver/install-cmake.sh centos:8 -RUN pkg-resolver/install-boost.sh centos:8 -RUN pkg-resolver/install-spotbugs.sh centos:8 -RUN pkg-resolver/install-protobuf.sh centos:8 -RUN pkg-resolver/install-zstandard.sh centos:8 +RUN pkg-resolver/install-maven.sh rockylinux:8 +RUN pkg-resolver/install-cmake.sh rockylinux:8 +RUN pkg-resolver/install-boost.sh rockylinux:8 +RUN pkg-resolver/install-spotbugs.sh rockylinux:8 +RUN pkg-resolver/install-protobuf.sh rockylinux:8 +RUN pkg-resolver/install-zstandard.sh rockylinux:8 RUN pkg-resolver/install-common-pkgs.sh + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc diff --git a/dev-support/docker/Dockerfile_ubuntu_24 b/dev-support/docker/Dockerfile_ubuntu_24 new file mode 100644 index 00000000000000..28ab04c3e09fe6 --- /dev/null +++ b/dev-support/docker/Dockerfile_ubuntu_24 @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM ubuntu:noble + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \ + && apt-get -q install -y --no-install-recommends python3 \ + && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ + && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \ + && apt-get -q update \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:noble) \ + && apt-get clean \ + && update-java-alternatives -s java-1.17.0-openjdk-amd64 \ + && rm -rf /var/lib/apt/lists/* + +RUN locale-gen en_US.UTF-8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +ENV PYTHONIOENCODING=utf-8 + +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME /usr +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64 + +####### +# Set env vars for SpotBugs 4.2.2 +####### +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Set env vars for Google Protobuf 3.21.12 +####### +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xms256m -Xmx3072m + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION true + +#### +# Install packages +#### +RUN pkg-resolver/install-spotbugs.sh ubuntu:noble +RUN pkg-resolver/install-boost.sh ubuntu:noble +RUN pkg-resolver/install-protobuf.sh ubuntu:noble +RUN pkg-resolver/install-hadolint.sh ubuntu:noble +RUN pkg-resolver/install-intel-isa-l.sh ubuntu:noble + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc diff --git a/dev-support/docker/Dockerfile_windows_10 b/dev-support/docker/Dockerfile_windows_10 index 105529c5d65b2f..7d32e6a84ff989 100644 --- a/dev-support/docker/Dockerfile_windows_10 +++ b/dev-support/docker/Dockerfile_windows_10 @@ -79,11 +79,11 @@ RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:T RUN powershell mkdir "C:\LibOpenSSL" RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C "C:\LibOpenSSL" -# Install libxxhash 0.8.1 needed for rsync 3.2.7. -RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar +# Install libxxhash 0.8.3 needed for rsync 3.2.7. +RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.3-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst +RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar RUN powershell mkdir "C:\LibXXHash" -RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash" +RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar -C "C:\LibXXHash" # Install libzstd 1.5.4 needed for rsync 3.2.7. RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst diff --git a/dev-support/docker/pkg-resolver/install-epel.sh b/dev-support/docker/pkg-resolver/install-epel.sh index 875dce3a9ae851..b6eabffca02be4 100644 --- a/dev-support/docker/pkg-resolver/install-epel.sh +++ b/dev-support/docker/pkg-resolver/install-epel.sh @@ -40,7 +40,7 @@ fi if [ "$version_to_install" == "8" ]; then mkdir -p /tmp/epel && - curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ + curl -L -s -S https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ -o /tmp/epel/epel-release-latest-8.noarch.rpm && rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm else diff --git a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh index 70e94b3792d9cf..81dd0002f40fb9 100644 --- a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh +++ b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh @@ -24,7 +24,7 @@ fi chmod a+x pkg-resolver/*.sh pkg-resolver/*.py chmod a+r pkg-resolver/*.json -if [ "$1" == "debian:10" ]; then +if [ "$1" == "debian:11" ]; then apt-get -q update apt-get -q install -y --no-install-recommends python3 \ python3-pip \ diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index 00d6ba36de21a6..b46d10e820027d 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -1,39 +1,39 @@ { "ant": { - "debian:10": "ant", + "debian:11": "ant", "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", - "centos:7": "ant", - "centos:8": "ant" + "ubuntu:noble": "ant", + "rockylinux:8": "ant" }, "apt-utils": { - "debian:10": "apt-utils", + "debian:11": "apt-utils", "ubuntu:focal": "apt-utils", + "ubuntu:noble": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, "automake": { - "debian:10": "automake", + "debian:11": "automake", "ubuntu:focal": "automake", + "ubuntu:noble": "automake", "ubuntu:focal::arch64": "automake", - "centos:7": "automake", - "centos:8": "automake" - }, - "autoconf": { - "centos:7": "autoconf" + "rockylinux:8": "automake" }, "bats": { - "debian:10": "bats", + "debian:11": "bats", "ubuntu:focal": "bats", - "ubuntu:focal::arch64": "bats" + "ubuntu:noble": "bats", + "ubuntu:focal::arch64": "bats", + "rockylinux:8": "bats" }, "build-essential": { - "debian:10": "build-essential", + "debian:11": "build-essential", "ubuntu:focal": "build-essential", - "ubuntu:focal::arch64": "build-essential", - "centos:7": "build-essential" + "ubuntu:noble": "build-essential", + "ubuntu:focal::arch64": "build-essential" }, "bzip2": { - "debian:10": [ + "debian:11": [ "bzip2", "libbz2-dev" ], @@ -41,32 +41,33 @@ "bzip2", "libbz2-dev" ], - "ubuntu:focal::arch64": [ + "ubuntu:noble": [ "bzip2", "libbz2-dev" ], - "centos:7": [ + "ubuntu:focal::arch64": [ "bzip2", - "bzip2-devel" + "libbz2-dev" ], - "centos:8": [ + "rockylinux:8": [ "bzip2", "bzip2-devel" ] }, "clang": { - "debian:10": "clang", + "debian:11": "clang", "ubuntu:focal": "clang", + "ubuntu:noble": "clang", "ubuntu:focal::arch64": "clang", - "centos:7": "clang", - "centos:8": "clang" + "rockylinux:8": "clang" }, "cmake": { "ubuntu:focal": "cmake", + "ubuntu:noble": "cmake", "ubuntu:focal::arch64": "cmake" }, "curl": { - "debian:10": [ + "debian:11": [ "curl", "libcurl4-openssl-dev" ], @@ -74,30 +75,30 @@ "curl", "libcurl4-openssl-dev" ], - "ubuntu:focal::arch64": [ + "ubuntu:noble": [ "curl", "libcurl4-openssl-dev" ], - "centos:7": [ + "ubuntu:focal::arch64": [ "curl", - "libcurl-devel" + "libcurl4-openssl-dev" ], - "centos:8": [ + "rockylinux:8": [ "curl", "libcurl-devel" ] }, "doxygen": { - "debian:10": "doxygen", + "debian:11": "doxygen", "ubuntu:focal": "doxygen", - "ubuntu:focal::arch64": "doxygen", - "centos:7": "doxygen" + "ubuntu:noble": "doxygen", + "ubuntu:focal::arch64": "doxygen" }, "dnf": { - "centos:8": "dnf" + "rockylinux:8": "dnf" }, "fuse": { - "debian:10": [ + "debian:11": [ "fuse", "libfuse-dev" ], @@ -105,23 +106,22 @@ "fuse", "libfuse-dev" ], - "ubuntu:focal::arch64": [ + "ubuntu:noble": [ "fuse", "libfuse-dev" ], - "centos:7": [ + "ubuntu:focal::arch64": [ "fuse", - "fuse-libs", - "fuse-devel" + "libfuse-dev" ], - "centos:8": [ + "rockylinux:8": [ "fuse", "fuse-libs", "fuse-devel" ] }, "gcc": { - "debian:10": { + "debian:11": { "bullseye": [ "gcc", "g++" @@ -131,61 +131,62 @@ "gcc", "g++" ], - "ubuntu:focal::arch64": [ + "ubuntu:noble": [ "gcc", "g++" ], - "centos:7": [ - "centos-release-scl", - "devtoolset-9" + "ubuntu:focal::arch64": [ + "gcc", + "g++" ] }, - "gettext": { - "centos:7": "gettext-devel" - }, "git": { - "debian:10": "git", + "debian:11": "git", "ubuntu:focal": "git", + "ubuntu:noble": "git", "ubuntu:focal::arch64": "git", - "centos:8": "git" + "rockylinux:8": "git" }, "gnupg-agent": { - "debian:10": "gnupg-agent", + "debian:11": "gnupg-agent", "ubuntu:focal": "gnupg-agent", + "ubuntu:noble": "gnupg-agent", "ubuntu:focal::arch64": "gnupg-agent" }, "hugo": { - "debian:10": "hugo", + "debian:11": "hugo", "ubuntu:focal": "hugo", + "ubuntu:noble": "hugo", "ubuntu:focal::arch64": "hugo" }, "libbcprov-java": { - "debian:10": "libbcprov-java", + "debian:11": "libbcprov-java", "ubuntu:focal": "libbcprov-java", + "ubuntu:noble": "libbcprov-java", "ubuntu:focal::arch64": "libbcprov-java" }, "libtool": { - "debian:10": "libtool", + "debian:11": "libtool", "ubuntu:focal": "libtool", + "ubuntu:noble": "libtool", "ubuntu:focal::arch64": "libtool", - "centos:7": "libtool", - "centos:8": "libtool" + "rockylinux:8": "libtool" }, "openssl": { - "debian:10": "libssl-dev", + "debian:11": "libssl-dev", "ubuntu:focal": "libssl-dev", + "ubuntu:noble": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", - "centos:7": "openssl-devel", - "centos:8": "openssl-devel" + "rockylinux:8": "openssl-devel" }, "perl": { - "centos:7": [ + "rockylinux:8": [ "perl-CPAN", "perl-devel" ] }, "protocol-buffers": { - "debian:10": [ + "debian:11": [ "libprotobuf-dev", "libprotoc-dev" ], @@ -193,26 +194,33 @@ "libprotobuf-dev", "libprotoc-dev" ], + "ubuntu:noble": [ + "libprotobuf-dev", + "libprotoc-dev" + ], "ubuntu:focal::arch64": [ "libprotobuf-dev", "libprotoc-dev" ] }, + "procps": { + "debian:11": "procps" + }, "sasl": { - "debian:10": "libsasl2-dev", + "debian:11": "libsasl2-dev", "ubuntu:focal": "libsasl2-dev", + "ubuntu:noble": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", - "centos:7": "cyrus-sasl-devel", - "centos:8": "cyrus-sasl-devel" + "rockylinux:8": "cyrus-sasl-devel" }, "snappy": { - "debian:10": "libsnappy-dev", + "debian:11": "libsnappy-dev", "ubuntu:focal": "libsnappy-dev", - "ubuntu:focal::arch64": "libsnappy-dev", - "centos:7": "snappy-devel" + "ubuntu:noble": "libsnappy-dev", + "ubuntu:focal::arch64": "libsnappy-dev" }, "zlib": { - "debian:10": [ + "debian:11": [ "libzstd-dev", "zlib1g-dev" ], @@ -220,71 +228,81 @@ "libzstd-dev", "zlib1g-dev" ], - "ubuntu:focal::arch64": [ + "ubuntu:noble": [ "libzstd-dev", "zlib1g-dev" ], - "centos:7": [ - "zlib-devel", - "lz4-devel" + "ubuntu:focal::arch64": [ + "libzstd-dev", + "zlib1g-dev" ], - "centos:8": [ + "rockylinux:8": [ "zlib-devel", "lz4-devel" ] }, "locales": { - "debian:10": "locales", + "debian:11": "locales", "ubuntu:focal": "locales", + "ubuntu:noble": "locales", "ubuntu:focal::arch64": "locales" }, "libtirpc-devel": { - "centos:7": "libtirpc-devel", - "centos:8": "libtirpc-devel" - }, - "libpmem": { - "centos:7": "libpmem-devel" + "rockylinux:8": "libtirpc-devel", + "ubuntu:noble": "libtirpc-dev" }, "make": { - "debian:10": "make", + "debian:11": "make", "ubuntu:focal": "make", + "ubuntu:noble": "make", "ubuntu:focal::arch64": "make", - "centos:7": "make", - "centos:8": "make" + "rockylinux:8": "make" }, "maven": { - "debian:10": "maven", + "debian:11": "maven", "ubuntu:focal": "maven", + "ubuntu:noble": "maven", "ubuntu:focal::arch64": "maven" }, "java": { - "debian:10": "openjdk-11-jdk", + "debian:11": [ + "openjdk-11-jdk", + "openjdk-17-jdk" + ], "ubuntu:focal": [ + "temurin-24-jdk", "openjdk-8-jdk", "openjdk-11-jdk", "openjdk-17-jdk" ], + "ubuntu:noble": [ + "temurin-24-jdk", + "openjdk-11-jdk", + "openjdk-17-jdk" + ], "ubuntu:focal::arch64": [ + "temurin-24-jdk", "openjdk-8-jdk", "openjdk-11-jdk", "openjdk-17-jdk" ] }, "pinentry-curses": { - "debian:10": "pinentry-curses", + "debian:11": "pinentry-curses", "ubuntu:focal": "pinentry-curses", + "ubuntu:noble": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", - "centos:7": "pinentry-curses", - "centos:8": "pinentry-curses" + "rockylinux:8": "pinentry-curses" }, "pkg-config": { - "debian:10": "pkg-config", + "debian:11": "pkg-config", "ubuntu:focal": "pkg-config", + "ubuntu:noble": "pkg-config", "ubuntu:focal::arch64": "pkg-config", - "centos:8": "pkg-config" + "rockylinux:8": "pkg-config" }, "python": { - "debian:10": [ + "debian:11": [ "python3", "python3-pip", "python3-pkg-resources", @@ -298,21 +316,22 @@ "python3-setuptools", "python3-wheel" ], - "ubuntu:focal::arch64": [ - "python2.7", + "ubuntu:noble": [ "python3", "python3-pip", "python3-pkg-resources", "python3-setuptools", "python3-wheel" ], - "centos:7": [ + "ubuntu:focal::arch64": [ + "python2.7", "python3", "python3-pip", + "python3-pkg-resources", "python3-setuptools", "python3-wheel" ], - "centos:8": [ + "rockylinux:8": [ "python3", "python3-pip", "python3-setuptools", @@ -320,43 +339,45 @@ ] }, "rsync": { - "debian:10": "rsync", + "debian:11": "rsync", "ubuntu:focal": "rsync", + "ubuntu:noble": "rsync", "ubuntu:focal::arch64": "rsync", - "centos:7": "rsync", - "centos:8": "rsync" + "rockylinux:8": "rsync" }, "shellcheck": { - "debian:10": "shellcheck", + "debian:11": "shellcheck", "ubuntu:focal": "shellcheck", + "ubuntu:noble": "shellcheck", "ubuntu:focal::arch64": "shellcheck" }, "shasum": { - "centos:7": "perl-Digest-SHA", - "centos:8": "perl-Digest-SHA" + "rockylinux:8": "perl-Digest-SHA" }, "software-properties-common": { - "debian:10": "software-properties-common", + "debian:11": "software-properties-common", "ubuntu:focal": "software-properties-common", + "ubuntu:noble": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" }, "sudo": { - "debian:10": "sudo", + "debian:11": "sudo", "ubuntu:focal": "sudo", + "ubuntu:noble": "sudo", "ubuntu:focal::arch64": "sudo", - "centos:7": "sudo", - "centos:8": "sudo" + "rockylinux:8": "sudo" }, "valgrind": { - "debian:10": "valgrind", + "debian:11": "valgrind", "ubuntu:focal": "valgrind", + "ubuntu:noble": "valgrind", "ubuntu:focal::arch64": "valgrind", - "centos:7": "valgrind", - "centos:8": "valgrind" + "rockylinux:8": "valgrind" }, "yasm": { - "debian:10": "yasm", + "debian:11": "yasm", "ubuntu:focal": "yasm", + "ubuntu:noble": "yasm", "ubuntu:focal::arch64": "yasm" } } diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index 93e2a93df42200..00f6c129b207d1 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -1,7 +1,7 @@ [ "ubuntu:focal", "ubuntu:focal::arch64", - "centos:7", - "centos:8", - "debian:10" -] \ No newline at end of file + "ubuntu:noble", + "rockylinux:8", + "debian:11" +] diff --git a/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh deleted file mode 100644 index 905ac5077deec2..00000000000000 --- a/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ $# -lt 1 ]; then - echo "ERROR: Need at least 1 argument, $# were provided" - exit 1 -fi - -if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then - cd /etc/yum.repos.d/ || exit && - sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && - sed -i 's|# *baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && - yum update -y && - cd /root || exit -else - echo "ERROR: Setting the archived baseurl is only supported for centos 7 and 8 environments" - exit 1 -fi diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3e79fe1b758486..18666e35c7e949 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -2762,6 +2762,16 @@ + + jdk17+ + + [17,) + + + 17 + ${javac.version} + + diff --git a/start-build-env.sh b/start-build-env.sh index 375a027a06b97a..b97eca8ad746e1 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -19,12 +19,29 @@ set -e # exit on error cd "$(dirname "$0")" # connect to root +OS_PLATFORM="${1:-}" +[ "$#" -gt 0 ] && shift + +DEFAULT_OS_PLATFORM="ubuntu_20" + +OS_PLATFORM_SUFFIX="" + +if [[ -n ${OS_PLATFORM} ]] && [[ "${OS_PLATFORM}" != "${DEFAULT_OS_PLATFORM}" ]]; then + # ubuntu_20 (default) platform does not have suffix in Dockerfile. + OS_PLATFORM_SUFFIX="_${OS_PLATFORM}" +fi + DOCKER_DIR=dev-support/docker DOCKER_FILE="${DOCKER_DIR}/Dockerfile" CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1) -if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then - DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64" +if [[ "$CPU_ARCH" == "aarch64" || "$CPU_ARCH" == "arm64" ]]; then + DOCKER_FILE="${DOCKER_DIR}/Dockerfile${OS_PLATFORM_SUFFIX}_aarch64" +fi + +if [ ! -e "${DOCKER_FILE}" ] ; then + echo "'${OS_PLATFORM}' environment not available yet for '${CPU_ARCH}'" + exit 1 fi docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR @@ -75,7 +92,7 @@ RUN rm -f /var/log/faillog /var/log/lastlog RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME} -d "${DOCKER_HOME_DIR}" RUN echo "${USER_NAME} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}" -ENV HOME "${DOCKER_HOME_DIR}" +ENV HOME="${DOCKER_HOME_DIR}" UserSpecificDocker @@ -93,4 +110,5 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \ -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \ -v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \ -u "${USER_ID}" \ - "hadoop-build-${USER_ID}" "$@" + --name "hadoop-build${OS_PLATFORM_SUFFIX}" \ + "hadoop-build${OS_PLATFORM_SUFFIX}-${USER_ID}" "$@"