From db9451928fa5b600be5c5da200e3ea44310f14ba Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Fri, 21 May 2021 15:36:03 +0530 Subject: [PATCH 1/3] HADOOP-17724. Add Dockerfile for Debian 10 * Adding a Dockerfile for building on Debian 10 since there are a lot of users in the community using this distro. --- dev-support/docker/Dockerfile_debian_10 | 201 ++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 dev-support/docker/Dockerfile_debian_10 diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_10 new file mode 100644 index 0000000000000..30bce15b62556 --- /dev/null +++ b/dev-support/docker/Dockerfile_debian_10 @@ -0,0 +1,201 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM debian:10 + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + +##### +# For installing the latest packages +##### +RUN echo 'deb http://deb.debian.org/debian testing main' >> /etc/apt/sources.list + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + +# hadolint ignore=DL3008 +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends \ + ant \ + apt-utils \ + bats \ + build-essential \ + bzip2 \ + clang \ + cmake \ + curl \ + doxygen \ + fuse \ + g++ \ + gcc \ + git \ + gnupg-agent \ + hugo \ + libbcprov-java \ + libbz2-dev \ + libcurl4-openssl-dev \ + libfuse-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libsasl2-dev \ + libsnappy-dev \ + libssl-dev \ + libtool \ + libzstd-dev \ + locales \ + make \ + maven \ + nodejs \ + node-yarn \ + npm \ + openjdk-11-jdk \ + pinentry-curses \ + pkg-config \ + python3 \ + python3-pip \ + python3-pkg-resources \ + python3-setuptools \ + python3-wheel \ + rsync \ + shellcheck \ + software-properties-common \ + sudo \ + valgrind \ + zlib1g-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME /usr +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 + +####### +# Install SpotBugs 4.2.2 +####### +RUN mkdir -p /opt/spotbugs \ + && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz \ + && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ + && chmod +x /opt/spotbugs/bin/* +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Install Boost 1.72 (1.71 ships with Focal) +####### +# hadolint ignore=DL3003 +RUN mkdir -p /opt/boost-library \ + && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ + && mv boost_1_72_0.tar.bz2 /opt/boost-library \ + && cd /opt/boost-library \ + && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ + && cd /opt/boost-library/boost_1_72_0 \ + && ./bootstrap.sh --prefix=/usr/ \ + && ./b2 --without-python install \ + && cd /root \ + && rm -rf /opt/boost-library + +###### +# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) +###### +# hadolint ignore=DL3003 +RUN mkdir -p /opt/protobuf-src \ + && curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz \ + && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ + && cd /opt/protobuf-src \ + && ./configure --prefix=/opt/protobuf \ + && make "-j$(nproc)" \ + && make install \ + && cd /root \ + && rm -rf /opt/protobuf-src +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + +#### +# Install pylint and python-dateutil +#### +RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 + +#### +# Install bower +#### +# hadolint ignore=DL3008 +RUN npm install -g bower@1.8.8 + +### +# Install hadolint +#### +RUN curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint \ + && chmod a+rx /bin/hadolint \ + && shasum -a 512 /bin/hadolint | \ + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' + +###### +# Intel ISA-L 2.29.0 +###### +# hadolint ignore=DL3003,DL3008 +RUN mkdir -p /opt/isa-l-src \ + && apt-get -q update \ + && apt-get install -y --no-install-recommends automake yasm \ + && apt-get clean \ + && curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz \ + && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ + && cd /opt/isa-l-src \ + && ./autogen.sh \ + && ./configure \ + && make "-j$(nproc)" \ + && make install \ + && cd /root \ + && rm -rf /opt/isa-l-src + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xms256m -Xmx3072m + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION true + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc From 9c95f906c26ba87951a0a01acc83261ac83e486b Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Fri, 21 May 2021 20:36:24 +0530 Subject: [PATCH 2/3] Use Java 11 * Set JAVA_HOME to Java JDK 11 since Java 8 isn't available. --- dev-support/docker/Dockerfile_debian_10 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_10 index 30bce15b62556..35abe6d956323 100644 --- a/dev-support/docker/Dockerfile_debian_10 +++ b/dev-support/docker/Dockerfile_debian_10 @@ -94,7 +94,7 @@ RUN apt-get -q update \ ###### ENV MAVEN_HOME /usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 ####### # Install SpotBugs 4.2.2 From 25813e19bd8f531e1d919ee8093399346aae7bf3 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 16 Jun 2021 10:51:54 +0530 Subject: [PATCH 3/3] Rebase as per pkg-resolver * Refactored the Dockerfile as per the tip of trunk. --- dev-support/docker/Dockerfile_debian_10 | 152 ++++-------------- dev-support/docker/pkg-resolver/packages.json | 64 ++++++++ .../docker/pkg-resolver/platforms.json | 3 +- 3 files changed, 95 insertions(+), 124 deletions(-) diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_10 index 35abe6d956323..8b7cbde7e624d 100644 --- a/dev-support/docker/Dockerfile_debian_10 +++ b/dev-support/docker/Dockerfile_debian_10 @@ -37,58 +37,27 @@ RUN echo 'deb http://deb.debian.org/debian testing main' >> /etc/apt/sources.lis ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true -# hadolint ignore=DL3008 +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - ant \ - apt-utils \ - bats \ - build-essential \ - bzip2 \ - clang \ - cmake \ - curl \ - doxygen \ - fuse \ - g++ \ - gcc \ - git \ - gnupg-agent \ - hugo \ - libbcprov-java \ - libbz2-dev \ - libcurl4-openssl-dev \ - libfuse-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libsasl2-dev \ - libsnappy-dev \ - libssl-dev \ - libtool \ - libzstd-dev \ - locales \ - make \ - maven \ - nodejs \ - node-yarn \ - npm \ - openjdk-11-jdk \ - pinentry-curses \ - pkg-config \ - python3 \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python3-wheel \ - rsync \ - shellcheck \ - software-properties-common \ - sudo \ - valgrind \ - zlib1g-dev \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +RUN locale-gen en_US.UTF-8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +ENV PYTHONIOENCODING=utf-8 + ###### # Set env vars required to build Hadoop ###### @@ -97,89 +66,16 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs 4.2.2 ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 (1.71 ships with Focal) +# Set env vars for Google Protobuf 3.7.1 ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 - -### -# Install hadolint -#### -RUN curl -L -s -S \ - https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ - -o /bin/hadolint \ - && chmod a+rx /bin/hadolint \ - && shasum -a 512 /bin/hadolint | \ - awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' - -###### -# Intel ISA-L 2.29.0 -###### -# hadolint ignore=DL3003,DL3008 -RUN mkdir -p /opt/isa-l-src \ - && apt-get -q update \ - && apt-get install -y --no-install-recommends automake yasm \ - && apt-get clean \ - && curl -L -s -S \ - https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ - -o /opt/isa-l.tar.gz \ - && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ - && cd /opt/isa-l-src \ - && ./autogen.sh \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/isa-l-src - ### # Avoid out of memory errors in builds ### @@ -188,6 +84,16 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m # Skip gpg verification when downloading Yetus via yetus-wrapper ENV HADOOP_SKIP_YETUS_VERIFICATION true +#### +# Install packages +#### +RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-spotbugs.sh debian:10 +RUN pkg-resolver/install-boost.sh debian:10 +RUN pkg-resolver/install-protobuf.sh debian:10 +RUN pkg-resolver/install-hadolint.sh debian:10 +RUN pkg-resolver/install-intel-isa-l.sh debian:10 + ### # Everything past this point is either not needed for testing or breaks Yetus. # So tell Yetus not to read the rest of the file: diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index 4ebbc900fe4de..cb4b7914d54d1 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -1,30 +1,39 @@ { "ant": { + "debian:10": "ant", "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", "centos:7": "ant", "centos:8": "ant" }, "apt-utils": { + "debian:10": "apt-utils", "ubuntu:focal": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, "automake": { + "debian:10": "automake", "ubuntu:focal": "automake", "ubuntu:focal::arch64": "automake", "centos:7": "automake", "centos:8": "automake" }, "bats": { + "debian:10": "bats", "ubuntu:focal": "bats", "ubuntu:focal::arch64": "bats" }, "build-essential": { + "debian:10": "build-essential", "ubuntu:focal": "build-essential", "ubuntu:focal::arch64": "build-essential", "centos:7": "build-essential" }, "bzip2": { + "debian:10": [ + "bzip2", + "libbz2-dev" + ], "ubuntu:focal": [ "bzip2", "libbz2-dev" @@ -43,16 +52,22 @@ ] }, "clang": { + "debian:10": "clang", "ubuntu:focal": "clang", "ubuntu:focal::arch64": "clang", "centos:7": "clang", "centos:8": "clang" }, "cmake": { + "debian:10": "cmake", "ubuntu:focal": "cmake", "ubuntu:focal::arch64": "cmake" }, "curl": { + "debian:10": [ + "curl", + "libcurl4-openssl-dev" + ], "ubuntu:focal": [ "curl", "libcurl4-openssl-dev" @@ -71,6 +86,7 @@ ] }, "doxygen": { + "debian:10": "doxygen", "ubuntu:focal": "doxygen", "ubuntu:focal::arch64": "doxygen", "centos:7": "doxygen" @@ -79,6 +95,10 @@ "centos:8": "dnf" }, "fuse": { + "debian:10": [ + "fuse", + "libfuse-dev" + ], "ubuntu:focal": [ "fuse", "libfuse-dev" @@ -99,6 +119,10 @@ ] }, "gcc": { + "debian:10": [ + "gcc", + "g++" + ], "ubuntu:focal": [ "gcc", "g++" @@ -113,36 +137,46 @@ ] }, "git": { + "debian:10": "git", "ubuntu:focal": "git", "ubuntu:focal::arch64": "git", "centos:7": "git", "centos:8": "git" }, "gnupg-agent": { + "debian:10": "gnupg-agent", "ubuntu:focal": "gnupg-agent", "ubuntu:focal::arch64": "gnupg-agent" }, "hugo": { + "debian:10": "hugo", "ubuntu:focal": "hugo", "ubuntu:focal::arch64": "hugo" }, "libbcprov-java": { + "debian:10": "libbcprov-java", "ubuntu:focal": "libbcprov-java", "ubuntu:focal::arch64": "libbcprov-java" }, "libtool": { + "debian:10": "libtool", "ubuntu:focal": "libtool", "ubuntu:focal::arch64": "libtool", "centos:7": "libtool", "centos:8": "libtool" }, "openssl": { + "debian:10": "libssl-dev", "ubuntu:focal": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", "centos:7": "openssl-devel", "centos:8": "openssl-devel" }, "protocol-buffers": { + "debian:10": [ + "libprotobuf-dev", + "libprotoc-dev" + ], "ubuntu:focal": [ "libprotobuf-dev", "libprotoc-dev" @@ -153,17 +187,23 @@ ] }, "sasl": { + "debian:10": "libsasl2-dev", "ubuntu:focal": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", "centos:7": "cyrus-sasl-devel", "centos:8": "cyrus-sasl-devel" }, "snappy": { + "debian:10": "libsnappy-dev", "ubuntu:focal": "libsnappy-dev", "ubuntu:focal::arch64": "libsnappy-dev", "centos:7": "snappy-devel" }, "zlib": { + "debian:10": [ + "libzstd-dev", + "zlib1g-dev" + ], "ubuntu:focal": [ "libzstd-dev", "zlib1g-dev" @@ -182,6 +222,7 @@ ] }, "locales": { + "debian:10": "locales", "ubuntu:focal": "locales", "ubuntu:focal::arch64": "locales" }, @@ -193,16 +234,23 @@ "centos:7": "libpmem-devel" }, "make": { + "debian:10": "make", "ubuntu:focal": "make", "ubuntu:focal::arch64": "make", "centos:7": "make", "centos:8": "make" }, "maven": { + "debian:10": "maven", "ubuntu:focal": "maven", "ubuntu:focal::arch64": "maven" }, "javascript": { + "debian:10": [ + "nodejs", + "node-yarn", + "npm" + ], "ubuntu:focal": [ "nodejs", "node-yarn", @@ -216,6 +264,7 @@ ] }, "java": { + "debian:10": "openjdk-11-jdk", "ubuntu:focal": [ "openjdk-8-jdk", "openjdk-11-jdk" @@ -226,17 +275,26 @@ ] }, "pinentry-curses": { + "debian:10": "pinentry-curses", "ubuntu:focal": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", "centos:7": "pinentry-curses", "centos:8": "pinentry-curses" }, "pkg-config": { + "debian:10": "pkg-config", "ubuntu:focal": "pkg-config", "ubuntu:focal::arch64": "pkg-config", "centos:8": "pkg-config" }, "python": { + "debian:10": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], "ubuntu:focal": [ "python3", "python3-pip", @@ -266,12 +324,14 @@ ] }, "rsync": { + "debian:10": "rsync", "ubuntu:focal": "rsync", "ubuntu:focal::arch64": "rsync", "centos:7": "rsync", "centos:8": "rsync" }, "shellcheck": { + "debian:10": "shellcheck", "ubuntu:focal": "shellcheck", "ubuntu:focal::arch64": "shellcheck" }, @@ -280,22 +340,26 @@ "centos:8": "perl-Digest-SHA" }, "software-properties-common": { + "debian:10": "software-properties-common", "ubuntu:focal": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" }, "sudo": { + "debian:10": "sudo", "ubuntu:focal": "sudo", "ubuntu:focal::arch64": "sudo", "centos:7": "sudo", "centos:8": "sudo" }, "valgrind": { + "debian:10": "valgrind", "ubuntu:focal": "valgrind", "ubuntu:focal::arch64": "valgrind", "centos:7": "valgrind", "centos:8": "valgrind" }, "yasm": { + "debian:10": "yasm", "ubuntu:focal": "yasm", "ubuntu:focal::arch64": "yasm" } diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index 4861c5dd55c00..93e2a93df4220 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -2,5 +2,6 @@ "ubuntu:focal", "ubuntu:focal::arch64", "centos:7", - "centos:8" + "centos:8", + "debian:10" ] \ No newline at end of file