HADOOP-19661. Migrate CentOS 8 to Rocky Linux 8 in build env Dockerfile

BUILDING.txt

@@ -31,7 +31,7 @@ On Linux / Mac:
 
     $ ./start-build-env.sh [OS platform]
 
-   - [OS Platform] One of [centos_7, centos_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
+   - [OS Platform] One of [rockylinux_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
         Default is 'ubuntu_20'.
         Note: Currently only default ('ubuntu_20') is supported on arm machine
 
@@ -461,7 +461,7 @@ path. This is necessary at least for Homebrewed OpenSSL.
 
 ----------------------------------------------------------------------------------
 
-Building on CentOS 8
+Building on Rocky Linux 8
 
 ----------------------------------------------------------------------------------
 
@@ -482,7 +482,7 @@ Building on CentOS 8
   $ sudo make install
   $ cd ..
 
-* Install libraries provided by CentOS 8.
+* Install libraries provided by Rocky Linux 8.
   $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
 
 * Install GCC 9.3.0

dev-support/Jenkinsfile

@@ -84,10 +84,10 @@ pipeline {
         stage ('setup sources') {
             steps {
 
-                dir("${WORKSPACE}/centos-8") {
+                dir("${WORKSPACE}/rockylinux-8") {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8
                     '''
                 }
 
@@ -111,11 +111,11 @@ pipeline {
         // C++/C++ build/platform.
         // This stage serves as a means of cross platform validation, which is
         // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Centos 8.
-        stage ('precommit-run Centos 8') {
+        // break the Hadoop build on Rocky Linux 8.
+        stage ('precommit-run Rocky Linux 8') {
             environment {
-                SOURCEDIR = "${WORKSPACE}/centos-8/src"
-                PATCHDIR = "${WORKSPACE}/centos-8/out"
+                SOURCEDIR = "${WORKSPACE}/rockylinux-8/src"
+                PATCHDIR = "${WORKSPACE}/rockylinux-8/out"
                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
                 IS_OPTIONAL = 1
             }
@@ -136,7 +136,7 @@ pipeline {
                 failure {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
+                    cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}"
                     '''
                     archiveArtifacts "out/**"
                 }

dev-support/docker/Dockerfile_centos_8

@@ -17,7 +17,7 @@
 # Dockerfile for installing the necessary dependencies for building Hadoop.
 # See BUILDING.txt.
 
-FROM centos:8
+FROM rockylinux:8
 
 WORKDIR /root
 
@@ -30,25 +30,18 @@ COPY pkg-resolver pkg-resolver
 RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
     && chmod a+r pkg-resolver/*.json
 
-######
-# Centos 8 has reached its EOL and the packages
-# are no longer available on mirror.centos.org site.
-# Please see https://www.centos.org/centos-linux-eol/
-######
-RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8
-
 ######
 # Install packages from yum
 ######
 # hadolint ignore=DL3008,SC2046
 RUN yum update -y \
     && yum install -y python3 \
-    && yum install -y $(pkg-resolver/resolve.py centos:8)
+    && yum install -y $(pkg-resolver/resolve.py rockylinux:8)
 
 ####
 # Install EPEL
 ####
-RUN pkg-resolver/install-epel.sh centos:8
+RUN pkg-resolver/install-epel.sh rockylinux:8
 
 RUN dnf --enablerepo=powertools install -y \
     doxygen \
@@ -69,50 +62,65 @@ SHELL ["/bin/bash", "--login", "-c"]
 # Set the environment variables needed for CMake
 # to find and use GCC 9 for compilation
 ######
-ENV GCC_HOME "/opt/rh/gcc-toolset-9"
-ENV CC "${GCC_HOME}/root/usr/bin/gcc"
-ENV CXX "${GCC_HOME}/root/usr/bin/g++"
-ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD"
-ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl"
-ENV SHLVL 1
-ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
-ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
-ENV MODULESHOME "/usr/share/Modules"
-ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
-ENV PCP_DIR "${GCC_HOME}/root"
-ENV MANPATH "${GCC_HOME}/root/usr/share/man::"
-ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
-ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
+ENV GCC_HOME="/opt/rh/gcc-toolset-9"
+ENV CC="${GCC_HOME}/root/usr/bin/gcc"
+ENV CXX="${GCC_HOME}/root/usr/bin/g++"
+ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD"
+ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl"
+ENV SHLVL=1
+ENV MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
+ENV MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
+ENV MODULESHOME="/usr/share/Modules"
+ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
+ENV PCP_DIR="${GCC_HOME}/root"
+ENV MANPATH="${GCC_HOME}/root/usr/share/man::"
+ENV PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig"
+ENV INFOPATH="${GCC_HOME}/root/usr/share/info"
 
 # TODO: Set locale
 
 ######
 # Set env vars required to build Hadoop
 ######
-ENV MAVEN_HOME /opt/maven
-ENV PATH "${PATH}:${MAVEN_HOME}/bin"
+ENV MAVEN_HOME=/opt/maven
+ENV PATH="${PATH}:${MAVEN_HOME}/bin"
 # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
+ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
 
 #######
 # Set env vars for SpotBugs
 #######
-ENV SPOTBUGS_HOME /opt/spotbugs
+ENV SPOTBUGS_HOME=/opt/spotbugs
 
 #######
 # Set env vars for Google Protobuf 3.21.12
 #######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
+ENV PROTOBUF_HOME=/opt/protobuf
+ENV PATH="${PATH}:${PROTOBUF_HOME}/bin"
+
+# Skip gpg verification when downloading Yetus via yetus-wrapper
+ENV HADOOP_SKIP_YETUS_VERIFICATION=true
 
 ######
 # Install packages
 ######
-RUN pkg-resolver/install-maven.sh centos:8
-RUN pkg-resolver/install-cmake.sh centos:8
-RUN pkg-resolver/install-boost.sh centos:8
-RUN pkg-resolver/install-spotbugs.sh centos:8
-RUN pkg-resolver/install-protobuf.sh centos:8
-RUN pkg-resolver/install-zstandard.sh centos:8
+RUN pkg-resolver/install-maven.sh rockylinux:8
+RUN pkg-resolver/install-cmake.sh rockylinux:8
+RUN pkg-resolver/install-boost.sh rockylinux:8
+RUN pkg-resolver/install-spotbugs.sh rockylinux:8
+RUN pkg-resolver/install-protobuf.sh rockylinux:8
+RUN pkg-resolver/install-zstandard.sh rockylinux:8
 RUN pkg-resolver/install-common-pkgs.sh
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
+
+# Add a welcome message and environment checks.
+COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
+RUN chmod 755 /root/hadoop_env_checks.sh
+# hadolint ignore=SC2016
+RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

dev-support/docker/pkg-resolver/install-epel.sh

@@ -40,7 +40,7 @@ fi
 
 if [ "$version_to_install" == "8" ]; then
   mkdir -p /tmp/epel &&
-    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
+    curl -L -s -S https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
       -o /tmp/epel/epel-release-latest-8.noarch.rpm &&
     rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
 else

dev-support/docker/pkg-resolver/packages.json

@@ -4,7 +4,7 @@
     "ubuntu:focal": "ant",
     "ubuntu:focal::arch64": "ant",
     "ubuntu:noble": "ant",
-    "centos:8": "ant"
+    "rockylinux:8": "ant"
   },
   "apt-utils": {
     "debian:10": "apt-utils",
@@ -17,7 +17,7 @@
     "ubuntu:focal": "automake",
     "ubuntu:noble": "automake",
     "ubuntu:focal::arch64": "automake",
-    "centos:8": "automake"
+    "rockylinux:8": "automake"
   },
   "bats": {
     "debian:10": "bats",
@@ -48,7 +48,7 @@
       "bzip2",
       "libbz2-dev"
     ],
-    "centos:8": [
+    "rockylinux:8": [
       "bzip2",
       "bzip2-devel"
     ]
@@ -58,7 +58,7 @@
     "ubuntu:focal": "clang",
     "ubuntu:noble": "clang",
     "ubuntu:focal::arch64": "clang",
-    "centos:8": "clang"
+    "rockylinux:8": "clang"
   },
   "cmake": {
     "ubuntu:focal": "cmake",
@@ -82,7 +82,7 @@
       "curl",
       "libcurl4-openssl-dev"
     ],
-    "centos:8": [
+    "rockylinux:8": [
       "curl",
       "libcurl-devel"
     ]
@@ -94,7 +94,7 @@
     "ubuntu:focal::arch64": "doxygen"
   },
   "dnf": {
-    "centos:8": "dnf"
+    "rockylinux:8": "dnf"
   },
   "fuse": {
     "debian:10": [
@@ -113,7 +113,7 @@
       "fuse",
       "libfuse-dev"
     ],
-    "centos:8": [
+    "rockylinux:8": [
       "fuse",
       "fuse-libs",
       "fuse-devel"
@@ -144,7 +144,7 @@
     "ubuntu:focal": "git",
     "ubuntu:noble": "git",
     "ubuntu:focal::arch64": "git",
-    "centos:8": "git"
+    "rockylinux:8": "git"
   },
   "gnupg-agent": {
     "debian:10": "gnupg-agent",
@@ -169,17 +169,17 @@
     "ubuntu:focal": "libtool",
     "ubuntu:noble": "libtool",
     "ubuntu:focal::arch64": "libtool",
-    "centos:8": "libtool"
+    "rockylinux:8": "libtool"
   },
   "openssl": {
     "debian:10": "libssl-dev",
     "ubuntu:focal": "libssl-dev",
     "ubuntu:noble": "libssl-dev",
     "ubuntu:focal::arch64": "libssl-dev",
-    "centos:8": "openssl-devel"
+    "rockylinux:8": "openssl-devel"
   },
   "perl": {
-    "centos:7": [
+    "rockylinux:7": [
       "perl-CPAN",
       "perl-devel"
     ]
@@ -207,7 +207,7 @@
     "ubuntu:focal": "libsasl2-dev",
     "ubuntu:noble": "libsasl2-dev",
     "ubuntu:focal::arch64": "libsasl2-dev",
-    "centos:8": "cyrus-sasl-devel"
+    "rockylinux:8": "cyrus-sasl-devel"
   },
   "snappy": {
     "debian:10": "libsnappy-dev",
@@ -232,7 +232,7 @@
       "libzstd-dev",
       "zlib1g-dev"
     ],
-    "centos:8": [
+    "rockylinux:8": [
       "zlib-devel",
       "lz4-devel"
     ]
@@ -244,15 +244,15 @@
     "ubuntu:focal::arch64": "locales"
   },
   "libtirpc-devel": {
-    "centos:8": "libtirpc-devel",
+    "rockylinux:8": "libtirpc-devel",
     "ubuntu:noble": "libtirpc-dev"
   },
   "make": {
     "debian:10": "make",
     "ubuntu:focal": "make",
     "ubuntu:noble": "make",
     "ubuntu:focal::arch64": "make",
-    "centos:8": "make"
+    "rockylinux:8": "make"
   },
   "maven": {
     "debian:10": "maven",
@@ -285,14 +285,14 @@
     "ubuntu:focal": "pinentry-curses",
     "ubuntu:noble": "pinentry-curses",
     "ubuntu:focal::arch64": "pinentry-curses",
-    "centos:8": "pinentry-curses"
+    "rockylinux:8": "pinentry-curses"
   },
   "pkg-config": {
     "debian:10": "pkg-config",
     "ubuntu:focal": "pkg-config",
     "ubuntu:noble": "pkg-config",
     "ubuntu:focal::arch64": "pkg-config",
-    "centos:8": "pkg-config"
+    "rockylinux:8": "pkg-config"
   },
   "python": {
     "debian:10": [
@@ -324,7 +324,7 @@
       "python3-setuptools",
       "python3-wheel"
     ],
-    "centos:8": [
+    "rockylinux:8": [
       "python3",
       "python3-pip",
       "python3-setuptools",
@@ -336,7 +336,7 @@
     "ubuntu:focal": "rsync",
     "ubuntu:noble": "rsync",
     "ubuntu:focal::arch64": "rsync",
-    "centos:8": "rsync"
+    "rockylinux:8": "rsync"
   },
   "shellcheck": {
     "debian:10": "shellcheck",
@@ -345,7 +345,7 @@
     "ubuntu:focal::arch64": "shellcheck"
   },
   "shasum": {
-    "centos:8": "perl-Digest-SHA"
+    "rockylinux:8": "perl-Digest-SHA"
   },
   "software-properties-common": {
     "debian:10": "software-properties-common",
@@ -358,14 +358,14 @@
     "ubuntu:focal": "sudo",
     "ubuntu:noble": "sudo",
     "ubuntu:focal::arch64": "sudo",
-    "centos:8": "sudo"
+    "rockylinux:8": "sudo"
   },
   "valgrind": {
     "debian:10": "valgrind",
     "ubuntu:focal": "valgrind",
     "ubuntu:noble": "valgrind",
     "ubuntu:focal::arch64": "valgrind",
-    "centos:8": "valgrind"
+    "rockylinux:8": "valgrind"
   },
   "yasm": {
     "debian:10": "yasm",