From 72298fa918ca0ca4fc26c7b8a8f0c7136150a8f5 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Fri, 4 Feb 2022 07:40:20 -0800 Subject: [PATCH 1/4] HDDS-6264. Bump centos to 8.4.2105 and some dependencies in ozone-runner --- Dockerfile | 65 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3bcad0e..8747368 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,34 +14,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.17.3-buster +FROM golang:1.17.6-buster RUN GO111MODULE=off go get -u github.com/rexray/gocsi/csc -FROM centos:7.6.1810 +FROM centos:8.4.2105 +RUN sed -i -e 's/^mirrorlist/#&/' -e 's/^#baseurl/baseurl/' -e 's/mirror.centos.org/vault.centos.org/' /etc/yum.repos.d/*.repo RUN yum -y install \ - bzip2-devel \ - gcc gcc-c++ gcc48-c++ \ - git \ - lz4-devel \ - make \ - snappy-devel \ - which \ - zlib-devel -RUN git clone https://github.com/gflags/gflags.git \ - && cd gflags \ - && git checkout v2.0 \ - && ./configure && make && make install -RUN curl -LSs -o zstd-1.1.3.tar.gz https://github.com/facebook/zstd/archive/v1.1.3.tar.gz \ - && tar zxvf zstd-1.1.3.tar.gz \ - && cd zstd-1.1.3 \ - && make && make install -RUN curl -LSs -o rocksdb-6.8.1.tar.gz https://github.com/facebook/rocksdb/archive/v6.8.1.tar.gz \ - && tar xzvf rocksdb-6.8.1.tar.gz \ - && cd rocksdb-6.8.1 \ - && make ldb - -FROM centos@sha256:b5e66c4651870a1ad435cd75922fe2cb943c9e973a9673822d1414824a1d0475 -RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + gcc gcc-c++ \ + git \ + make \ + which \ + cmake +RUN curl -LSs -o gflags-src.tar.gz https://github.com/gflags/gflags/archive/v2.2.2.tar.gz \ + && tar zxvf gflags-src.tar.gz \ + && cd gflags-2.2.2 \ + && mkdir build \ + && cd build \ + && cmake .. \ + && make -j$(nproc) \ + && make install \ + && cd ../.. \ + && rm -rf gflags-2.2.2 +RUN curl -LSs -o zstd-src.tar.gz https://github.com/facebook/zstd/archive/v1.5.2.tar.gz \ + && tar zxvf zstd-src.tar.gz \ + && cd zstd-1.5.2 \ + && make -j$(nproc) \ + && make install \ + && cd .. \ + && rm -rf zstd-1.5.2 +RUN curl -LSs -o rocksdb-src.tar.gz https://github.com/facebook/rocksdb/archive/v6.28.2.tar.gz \ + && tar xzvf rocksdb-src.tar.gz \ + && cd rocksdb-6.28.2 \ + && make -j$(nproc) ldb + +FROM centos:8.4.2105 +RUN sed -i -e 's/^mirrorlist/#&/' -e 's/^#baseurl/baseurl/' -e 's/mirror.centos.org/vault.centos.org/' /etc/yum.repos.d/*.repo +RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm RUN yum install -y \ awscli \ bzip2 \ @@ -53,14 +61,15 @@ RUN yum install -y \ sudo \ wget \ zlib +# Create python symlink for compatiblity +RUN ln -s /usr/bin/python3 /usr/bin/python COPY --from=0 /go/bin/csc /usr/bin/csc -COPY --from=1 /rocksdb-6.8.1/ldb /usr/local/bin/ldb +COPY --from=1 /rocksdb-6.28.2/ldb /usr/local/bin/ldb COPY --from=1 /usr/local/lib /usr/local/lib/ #For executing inline smoketest -RUN pip3 install robotframework -RUN pip3 install boto3 +RUN pip3 install robotframework boto3 #dumb init for proper init handling RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 From 1571b4be1553d11462713ba0c3444bd6379db3dc Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Sat, 5 Feb 2022 07:32:45 -0800 Subject: [PATCH 2/4] Use latest awscli (old version in centos 8.4 repo is causing issue); Install diffutils for diff command; Remove /run/nologin to allow ssh as non-root user. --- Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 38a0fd2..c3a1078 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,6 @@ FROM centos:8.4.2105 RUN sed -i -e 's/^mirrorlist/#&/' -e 's/^#baseurl/baseurl/' -e 's/mirror.centos.org/vault.centos.org/' /etc/yum.repos.d/*.repo RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm RUN yum install -y \ - awscli \ bzip2 \ java-11-openjdk \ jq \ @@ -60,7 +59,8 @@ RUN yum install -y \ snappy \ sudo \ wget \ - zlib + zlib \ + diffutils RUN sudo python3 -m pip install --upgrade pip # Create python symlink for compatiblity RUN ln -s /usr/bin/python3 /usr/bin/python @@ -70,7 +70,7 @@ COPY --from=1 /rocksdb-6.28.2/ldb /usr/local/bin/ldb COPY --from=1 /usr/local/lib /usr/local/lib/ #For executing inline smoketest -RUN pip3 install robotframework boto3 +RUN pip3 install awscli robotframework boto3 #dumb init for proper init handling RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 @@ -117,6 +117,9 @@ RUN mkdir /data && chmod 1777 /data ADD entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod 755 /usr/local/bin/entrypoint.sh +# Fix error when ssh'ing into other containers: System is booting up. Unprivileged users are not permitted to log in yet. +RUN rm -f /run/nologin + WORKDIR /opt/hadoop USER hadoop From a69a55444f3f80bf72c4c7e31b8cca7dc702cfbf Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Sun, 6 Feb 2022 21:48:07 -0800 Subject: [PATCH 3/4] Use centos:7.9.2009 as the base image; Remove unused git package; dumb-init 1.2.5 (was 1.2.0); byteman 4.0.9 (was 4.0.4); async-profiler-2.6 (was 2.0); goofys 0.24.0 (was 0.20.0) --- Dockerfile | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index c3a1078..fd626f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,14 +17,15 @@ FROM golang:1.17.6-buster RUN GO111MODULE=off go get -u github.com/rexray/gocsi/csc -FROM centos:8.4.2105 -RUN sed -i -e 's/^mirrorlist/#&/' -e 's/^#baseurl/baseurl/' -e 's/mirror.centos.org/vault.centos.org/' /etc/yum.repos.d/*.repo +FROM centos:7.9.2009 +# Required for cmake3 package +RUN yum -y install epel-release RUN yum -y install \ gcc gcc-c++ \ - git \ make \ which \ - cmake + cmake3 +RUN ln -s /usr/bin/cmake3 /usr/bin/cmake RUN curl -LSs -o gflags-src.tar.gz https://github.com/gflags/gflags/archive/v2.2.2.tar.gz \ && tar zxvf gflags-src.tar.gz \ && cd gflags-2.2.2 \ @@ -47,9 +48,8 @@ RUN curl -LSs -o rocksdb-src.tar.gz https://github.com/facebook/rocksdb/archive/ && cd rocksdb-6.28.2 \ && make -j$(nproc) ldb -FROM centos:8.4.2105 -RUN sed -i -e 's/^mirrorlist/#&/' -e 's/^#baseurl/baseurl/' -e 's/mirror.centos.org/vault.centos.org/' /etc/yum.repos.d/*.repo -RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm +FROM centos:7.9.2009 +RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm RUN yum install -y \ bzip2 \ java-11-openjdk \ @@ -62,8 +62,6 @@ RUN yum install -y \ zlib \ diffutils RUN sudo python3 -m pip install --upgrade pip -# Create python symlink for compatiblity -RUN ln -s /usr/bin/python3 /usr/bin/python COPY --from=0 /go/bin/csc /usr/bin/csc COPY --from=1 /rocksdb-6.28.2/ldb /usr/local/bin/ldb @@ -73,17 +71,17 @@ COPY --from=1 /usr/local/lib /usr/local/lib/ RUN pip3 install awscli robotframework boto3 #dumb init for proper init handling -RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 +RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 RUN chmod +x /usr/local/bin/dumb-init #byteman test for development -ADD https://repo.maven.apache.org/maven2/org/jboss/byteman/byteman/4.0.4/byteman-4.0.4.jar /opt/byteman.jar +ADD https://repo.maven.apache.org/maven2/org/jboss/byteman/byteman/4.0.9/byteman-4.0.9.jar /opt/byteman.jar RUN chmod o+r /opt/byteman.jar #async profiler for development profiling RUN cd /opt && \ - curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.0/async-profiler-2.0-linux-x64.tar.gz | tar xvz && \ - mv async-profiler-2.0-linux-x64 profiler + curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.6/async-profiler-2.6-linux-x64.tar.gz | tar xvz && \ + mv async-profiler-2.6-linux-x64 profiler ENV JAVA_HOME=/usr/lib/jvm/jre/ ENV LD_LIBRARY_PATH /usr/local/lib @@ -103,7 +101,7 @@ RUN chmod 644 /etc/krb5.conf RUN yum install -y krb5-workstation # CSI / k8s / fuse / goofys dependency -RUN wget https://github.com/kahing/goofys/releases/download/v0.20.0/goofys -O /usr/bin/goofys +RUN wget https://github.com/kahing/goofys/releases/download/v0.24.0/goofys -O /usr/bin/goofys RUN chmod 755 /usr/bin/goofys RUN yum install -y fuse @@ -117,9 +115,6 @@ RUN mkdir /data && chmod 1777 /data ADD entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod 755 /usr/local/bin/entrypoint.sh -# Fix error when ssh'ing into other containers: System is booting up. Unprivileged users are not permitted to log in yet. -RUN rm -f /run/nologin - WORKDIR /opt/hadoop USER hadoop From 9eb62afda1b4d66151be74abd06f862fd9b7c7ac Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Sun, 6 Feb 2022 23:26:06 -0800 Subject: [PATCH 4/4] Add optional rocksdb build dependency perl; Fix comments --- Dockerfile | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd626f4..a9bed30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,28 +24,36 @@ RUN yum -y install \ gcc gcc-c++ \ make \ which \ - cmake3 + cmake3 \ + perl RUN ln -s /usr/bin/cmake3 /usr/bin/cmake -RUN curl -LSs -o gflags-src.tar.gz https://github.com/gflags/gflags/archive/v2.2.2.tar.gz \ +RUN export GFLAGS_VER=2.2.2 \ + && curl -LSs -o gflags-src.tar.gz https://github.com/gflags/gflags/archive/v${GFLAGS_VER}.tar.gz \ && tar zxvf gflags-src.tar.gz \ - && cd gflags-2.2.2 \ + && rm gflags-src.tar.gz \ + && cd gflags-${GFLAGS_VER} \ && mkdir build \ && cd build \ && cmake .. \ && make -j$(nproc) \ && make install \ && cd ../.. \ - && rm -rf gflags-2.2.2 -RUN curl -LSs -o zstd-src.tar.gz https://github.com/facebook/zstd/archive/v1.5.2.tar.gz \ + && rm -rf gflags-${GFLAGS_VER} +RUN export ZSTD_VER=1.5.2 \ + && curl -LSs -o zstd-src.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VER}.tar.gz \ && tar zxvf zstd-src.tar.gz \ - && cd zstd-1.5.2 \ + && rm zstd-src.tar.gz \ + && cd zstd-${ZSTD_VER} \ && make -j$(nproc) \ && make install \ && cd .. \ - && rm -rf zstd-1.5.2 -RUN curl -LSs -o rocksdb-src.tar.gz https://github.com/facebook/rocksdb/archive/v6.28.2.tar.gz \ + && rm -rf zstd-${ZSTD_VER} +RUN export ROCKSDB_VER=6.28.2 \ + && curl -LSs -o rocksdb-src.tar.gz https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VER}.tar.gz \ && tar xzvf rocksdb-src.tar.gz \ - && cd rocksdb-6.28.2 \ + && rm rocksdb-src.tar.gz \ + && mv rocksdb-${ROCKSDB_VER} rocksdb \ + && cd rocksdb \ && make -j$(nproc) ldb FROM centos:7.9.2009 @@ -64,7 +72,7 @@ RUN yum install -y \ RUN sudo python3 -m pip install --upgrade pip COPY --from=0 /go/bin/csc /usr/bin/csc -COPY --from=1 /rocksdb-6.28.2/ldb /usr/local/bin/ldb +COPY --from=1 /rocksdb/ldb /usr/local/bin/ldb COPY --from=1 /usr/local/lib /usr/local/lib/ #For executing inline smoketest @@ -94,7 +102,7 @@ RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers RUN chown hadoop /opt -#Be prepared for kerbebrizzed cluster +# Prep for Kerberized cluster RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs ADD krb5.conf /etc/ RUN chmod 644 /etc/krb5.conf @@ -105,13 +113,13 @@ RUN wget https://github.com/kahing/goofys/releases/download/v0.24.0/goofys -O /u RUN chmod 755 /usr/bin/goofys RUN yum install -y fuse -#Make it compatible with any UID/GID (write premission may be missing to /opt/hadoop +# Create hadoop and data directories. Grant all permission to all on them RUN mkdir -p /etc/hadoop && mkdir -p /var/log/hadoop && chmod 1777 /etc/hadoop && chmod 1777 /var/log/hadoop ENV OZONE_LOG_DIR=/var/log/hadoop ENV OZONE_CONF_DIR=/etc/hadoop RUN mkdir /data && chmod 1777 /data -#default entrypoint (used only if the ozone dir is not bindmounted) +# Set default entrypoint (used only if the ozone dir is not bind mounted) ADD entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod 755 /usr/local/bin/entrypoint.sh