diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index e260fe1638c..07a4328bd88 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -28,7 +28,7 @@ UTF-8 true - 20240316-jdk17-1 + 20240712-jdk17-2 apache/ozone-testkrb5:20230318-1 true diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml b/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml index 77f02f7286d..8600659b786 100644 --- a/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml +++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml @@ -18,6 +18,7 @@ services: rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} hostname: rm + dns_search: . volumes: - ../..:/opt/ozone - ../../libexec/transformation.py:/opt/transformation.py @@ -36,6 +37,7 @@ services: nm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} hostname: nm + dns_search: . volumes: - ../..:/opt/ozone - ../../libexec/transformation.py:/opt/transformation.py @@ -55,6 +57,7 @@ services: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} container_name: jhs hostname: jhs + dns_search: . volumes: - ../..:/opt/ozone - ../_keytabs:/etc/security/keytabs diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/Dockerfile index 714a6f56d66..55fa7323461 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/Dockerfile +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/Dockerfile @@ -20,13 +20,14 @@ ARG OZONE_RUNNER_VERSION FROM ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} # Install ssh -RUN sudo yum install -y openssh-clients openssh-server +RUN sudo dnf install -y openssh-clients openssh-server RUN sudo ssh-keygen -A RUN sudo mkdir -p /run/sshd RUN sudo sed -i "s/.*UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config RUN sudo sed -i "s/.*PermitUserEnvironment.*/PermitUserEnvironment yes/g" /etc/ssh/sshd_config RUN sudo sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd +RUN sudo rm -f /etc/nologin /var/run/nologin #/opt/hadoop is mounted, we can't use it as a home RUN sudo usermod -d /opt hadoop @@ -38,6 +39,8 @@ RUN sudo chmod 700 /opt/.ssh RUN echo "export JAVA_HOME=$JAVA_HOME" | sudo sh -c 'cat >> /etc/profile' RUN echo "JAVA_HOME=$JAVA_HOME" | sh -c 'cat >> /opt/.ssh/environment' +RUN echo "OZONE_CONF_DIR=$OZONE_CONF_DIR" >> /opt/.ssh/environment +RUN echo "OZONE_LOG_DIR=$OZONE_LOG_DIR" >> /opt/.ssh/environment # Install required robot framework libraries RUN sudo pip3 install robotframework-sshlibrary diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile index 35c6ccabd37..f7494a70b39 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile +++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile @@ -19,13 +19,14 @@ ARG OZONE_RUNNER_VERSION FROM ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} -RUN sudo yum install -y openssh-clients openssh-server +RUN sudo dnf install -y openssh-clients openssh-server RUN sudo ssh-keygen -A RUN sudo mkdir -p /run/sshd RUN sudo sed -i "s/.*UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config RUN sudo sed -i "s/.*PermitUserEnvironment.*/PermitUserEnvironment yes/g" /etc/ssh/sshd_config RUN sudo sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd +RUN sudo rm -f /etc/nologin /var/run/nologin #/opt/hadoop is mounted, we can't use it as a home RUN sudo usermod -d /opt hadoop @@ -36,6 +37,8 @@ RUN sudo chmod 600 /opt/.ssh/* RUN sudo chmod 700 /opt/.ssh RUN echo "export JAVA_HOME=$JAVA_HOME" | sudo sh -c 'cat >> /etc/profile' -RUN echo "JAVA_HOME=$JAVA_HOME" | sh -c 'cat >> /opt/.ssh/environment' +RUN echo "JAVA_HOME=$JAVA_HOME" >> /opt/.ssh/environment +RUN echo "OZONE_CONF_DIR=$OZONE_CONF_DIR" >> /opt/.ssh/environment +RUN echo "OZONE_LOG_DIR=$OZONE_LOG_DIR" >> /opt/.ssh/environment CMD ["sudo","/usr/sbin/sshd","-D"] diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/disabled-test.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/test.sh similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozonescripts/disabled-test.sh rename to hadoop-ozone/dist/src/main/compose/ozonescripts/test.sh diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml index 4ef826c8147..d55d5e0e2e8 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml @@ -18,6 +18,7 @@ services: kdc: image: ${OZONE_TESTKRB5_IMAGE} hostname: kdc + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -27,6 +28,7 @@ services: ipv4_address: 172.25.0.100 kms: image: apache/hadoop:${HADOOP_VERSION} + dns_search: . ports: - 9600:9600 env_file: @@ -43,6 +45,7 @@ services: ipv4_address: 172.25.0.101 datanode1: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -65,6 +68,7 @@ services: ipv4_address: 172.25.0.102 datanode2: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -87,6 +91,7 @@ services: ipv4_address: 172.25.0.103 datanode3: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -110,6 +115,7 @@ services: om1: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: om1 + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -135,6 +141,7 @@ services: om2: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: om2 + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -160,6 +167,7 @@ services: om3: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: om3 + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -185,6 +193,7 @@ services: httpfs: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: httpfs + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -203,6 +212,7 @@ services: s3g: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: s3g + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -220,6 +230,7 @@ services: scm1.org: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: scm1.org + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -247,6 +258,7 @@ services: scm2.org: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: scm2.org + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -275,6 +287,7 @@ services: scm3.org: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: scm3.org + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -303,6 +316,7 @@ services: recon: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: recon + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml index e2a240eed77..625aac77723 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml @@ -18,12 +18,14 @@ services: kdc: image: ${OZONE_TESTKRB5_IMAGE} hostname: kdc + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs command: ["krb5kdc","-n"] kms: image: apache/hadoop:3 + dns_search: . ports: - 9600:9600 env_file: @@ -34,6 +36,7 @@ services: command: ["hadoop", "kms"] datanode: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -48,6 +51,7 @@ services: om: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: om + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -64,6 +68,7 @@ services: s3g: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: s3g + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -78,6 +83,7 @@ services: scm: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: scm + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml index a9816700c29..39d26c362f6 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml @@ -18,6 +18,7 @@ services: kdc: image: ${OZONE_TESTKRB5_IMAGE} hostname: kdc + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -25,6 +26,7 @@ services: kms: image: apache/hadoop:${HADOOP_VERSION} hostname: kms + dns_search: . ports: - 9600:9600 env_file: @@ -37,6 +39,7 @@ services: datanode: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: dn + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -51,6 +54,7 @@ services: om: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: om + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -67,6 +71,7 @@ services: httpfs: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: httpfs + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -82,6 +87,7 @@ services: s3g: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: s3g + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -96,6 +102,7 @@ services: recon: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: recon + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs @@ -110,6 +117,7 @@ services: scm: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: scm + dns_search: . volumes: - ../..:/opt/hadoop - ../_keytabs:/etc/security/keytabs diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml index eb79225aa61..84ae48fbbc3 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml @@ -21,6 +21,7 @@ x-common-config: - docker-config - ../../../common/security.conf image: ${OZONE_IMAGE} + dns_search: . x-environment: &environment