diff --git a/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.yaml b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.yaml index 8ddf3f5f65a3..aa5087cbfd23 100644 --- a/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.yaml +++ b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.yaml @@ -34,13 +34,7 @@ services: command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] s3g1: <<: *s3-worker - ports: - - 9879:9878 s3g2: <<: *s3-worker - ports: - - 9880:9878 s3g3: <<: *s3-worker - ports: - - 9881:9878 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh similarity index 75% rename from hadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh rename to hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh index 6cf3901b9d28..af67a7099dde 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/disabled-test-s3-haproxy.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh @@ -15,11 +15,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +#suite:HA-unsecure + COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR export SECURITY_ENABLED=false export COMPOSE_FILE=docker-compose.yaml:../common/s3-haproxy.yaml +export OZONE_REPLICATION_FACTOR=3 +export SCM=scm1 # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" @@ -27,4 +31,10 @@ source "$COMPOSE_DIR/../testlib.sh" start_docker_env ## Exclude virtual-host tests. This is tested separately as it requires additional config. -execute_robot_test scm --exclude virtual-host s3 +exclude="--exclude virtual-host" +for bucket in generated; do + execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + # some tests are independent of the bucket type, only need to be run once + ## Exclude awss3virtualhost.robot + exclude="--exclude virtual-host --exclude no-bucket-type" +done diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml index e48d3cb9b050..425875d7cfd9 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml @@ -14,6 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +x-common-config: + &common-config + dns_search: . + image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + env_file: + - docker-config + services: kdc: image: ${OZONE_TESTKRB5_IMAGE} @@ -44,12 +55,7 @@ services: ozone_net: ipv4_address: 172.25.0.101 datanode1: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 19864:9999 command: ["/opt/hadoop/bin/ozone","datanode"] @@ -58,8 +64,6 @@ services: - "scm2.org: 172.25.0.117" - "scm3.org: 172.25.0.118" - "recon: 172.25.0.115" - env_file: - - docker-config environment: WAITFOR: scm3.org:9894 OZONE_OPTS: @@ -67,12 +71,7 @@ services: ozone_net: ipv4_address: 172.25.0.102 datanode2: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9866:9999 command: ["/opt/hadoop/bin/ozone","datanode"] @@ -81,8 +80,6 @@ services: - "scm2.org: 172.25.0.117" - "scm3.org: 172.25.0.118" - "recon: 172.25.0.115" - env_file: - - docker-config environment: WAITFOR: scm3.org:9894 OZONE_OPTS: @@ -90,12 +87,7 @@ services: ozone_net: ipv4_address: 172.25.0.103 datanode3: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9868:9999 command: ["/opt/hadoop/bin/ozone","datanode"] @@ -104,8 +96,6 @@ services: - "scm2.org: 172.25.0.117" - "scm3.org: 172.25.0.118" - "recon: 172.25.0.115" - env_file: - - docker-config environment: WAITFOR: scm3.org:9894 OZONE_OPTS: @@ -113,13 +103,8 @@ services: ozone_net: ipv4_address: 172.25.0.104 om1: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: om1 - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9880:9874 - 9890:9872 @@ -128,8 +113,6 @@ services: WAITFOR: scm3.org:9894 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION OZONE_OPTS: - env_file: - - ./docker-config command: ["/opt/hadoop/bin/ozone","om"] extra_hosts: - "scm1.org: 172.25.0.116" @@ -139,13 +122,8 @@ services: ozone_net: ipv4_address: 172.25.0.111 om2: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: om2 - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9882:9874 - 9892:9872 @@ -154,8 +132,6 @@ services: WAITFOR: scm3.org:9894 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION OZONE_OPTS: - env_file: - - ./docker-config command: ["/opt/hadoop/bin/ozone","om"] extra_hosts: - "scm1.org: 172.25.0.116" @@ -165,13 +141,8 @@ services: ozone_net: ipv4_address: 172.25.0.112 om3: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: om3 - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9884:9874 - 9894:9872 @@ -180,8 +151,6 @@ services: WAITFOR: scm3.org:9894 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION OZONE_OPTS: - env_file: - - ./docker-config command: ["/opt/hadoop/bin/ozone","om"] extra_hosts: - "scm1.org: 172.25.0.116" @@ -191,17 +160,10 @@ services: ozone_net: ipv4_address: 172.25.0.113 httpfs: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: httpfs - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 14000:14000 - env_file: - - ./docker-config command: [ "/opt/hadoop/bin/ozone","httpfs" ] environment: OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} @@ -210,31 +172,15 @@ services: ozone_net: ipv4_address: 172.25.0.119 s3g: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: s3g - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf - ports: - - 9878:9878 - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","s3g"] - environment: - OZONE_OPTS: + <<: *common-config + command: ["ozone","s3g"] networks: ozone_net: - ipv4_address: 172.25.0.114 + ipv4_address: 172.25.0.120 scm1.org: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: scm1.org - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9990:9876 - 9992:9860 @@ -256,18 +202,11 @@ services: ozone_net: ipv4_address: 172.25.0.116 scm2.org: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: scm2.org - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9994:9876 - 9996:9860 - env_file: - - docker-config environment: WAITFOR: scm1.org:9894 ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION @@ -285,18 +224,11 @@ services: ozone_net: ipv4_address: 172.25.0.117 scm3.org: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: scm3.org - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9998:9876 - 10002:9860 - env_file: - - docker-config environment: WAITFOR: scm2.org:9894 ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION @@ -314,17 +246,10 @@ services: ozone_net: ipv4_address: 172.25.0.118 recon: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} hostname: recon - dns_search: . - volumes: - - ../..:/opt/hadoop - - ../_keytabs:/etc/security/keytabs - - ./krb5.conf:/etc/krb5.conf + <<: *common-config ports: - 9888:9888 - env_file: - - ./docker-config environment: OZONE_OPTS: command: ["/opt/hadoop/bin/ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.cfg b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.cfg new file mode 100644 index 000000000000..c93d1b16b64d --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.cfg @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Simple configuration for an HTTP proxy listening on port 9878 on all +# interfaces and forwarding requests to a multiple multiple S3 servers in round +# robin fashion. +global + daemon + maxconn 256 + +defaults + mode http + timeout connect 5000ms + timeout client 50000ms + timeout server 50000ms + +frontend http-in + bind *:9878 + default_backend servers + +backend servers + balance roundrobin + server server1 172.25.0.121:9878 maxconn 32 + server server2 172.25.0.122:9878 maxconn 32 + server server3 172.25.0.123:9878 maxconn 32 + +frontend webadmin + bind *:19878 + default_backend webadmin-servers + +backend webadmin-servers + balance roundrobin + server server1 172.25.0.121:19878 maxconn 32 + server server2 172.25.0.122:19878 maxconn 32 + server server3 172.25.0.123:19878 maxconn 32 diff --git a/hadoop-ozone/dist/src/main/compose/common/s3-haproxy-secure.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.yaml similarity index 67% rename from hadoop-ozone/dist/src/main/compose/common/s3-haproxy-secure.yaml rename to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.yaml index 9e6fee1ac40c..b549426c7d8a 100644 --- a/hadoop-ozone/dist/src/main/compose/common/s3-haproxy-secure.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.yaml @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -x-s3-worker: - &s3-worker +x-common-config: + &common-config + dns_search: . image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} volumes: - ../..:/opt/hadoop @@ -23,35 +24,41 @@ x-s3-worker: - ./krb5.conf:/etc/krb5.conf env_file: - docker-config - command: ["ozone","s3g", "-Dozone.om.transport.class=${OZONE_S3_OM_TRANSPORT:-org.apache.hadoop.ozone.om.protocolPB.GrpcOmTransportFactory}"] services: + s3g1: + hostname: s3g1 + <<: *common-config + command: ["ozone","s3g"] + networks: + ozone_net: + ipv4_address: 172.25.0.121 + s3g2: + hostname: s3g2 + <<: *common-config + command: ["ozone","s3g"] + networks: + ozone_net: + ipv4_address: 172.25.0.122 + s3g3: + hostname: s3g3 + <<: *common-config + command: ["ozone","s3g"] + networks: + ozone_net: + ipv4_address: 172.25.0.123 s3g: image: haproxy:lts-alpine hostname: s3g dns_search: . volumes: - ../..:/opt/hadoop - - ../common/s3-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf + - ./s3-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg ports: - 9878:9878 command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] - s3g1: - <<: *s3-worker - hostname: s3g1 - dns_search: . - ports: - - 9879:9878 - s3g2: - - <<: *s3-worker - hostname: s3g2 - dns_search: . - ports: - - 9880:9878 - s3g3: - <<: *s3-worker - hostname: s3g3 - dns_search: . - ports: - - 9881:9878 + networks: + ozone_net: + ipv4_address: 172.25.0.120 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-port-forward.yml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-port-forward.yml new file mode 100644 index 000000000000..08ec7d3a92e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-port-forward.yml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + s3g: + ports: + - 9878:9878 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml index 23f3728e9c33..b4adac7bd58b 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml @@ -40,6 +40,6 @@ services: s3g: <<: *s3g-virtual-host-config extra_hosts: - - "bucket1.s3g.internal: 172.25.0.114" + - "bucket1.s3g.internal: 172.25.0.120" recon: <<: *s3g-virtual-host-config diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/scm-decommission.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/scm-decommission.yaml index 7964bdff9f8e..c7e53f9651eb 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/scm-decommission.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/scm-decommission.yaml @@ -41,7 +41,7 @@ services: - "scm3.org: 172.25.0.118" networks: ozone_net: - ipv4_address: 172.25.0.120 + ipv4_address: 172.25.0.220 profiles: ["scm4.org"] datanode4: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} @@ -56,7 +56,7 @@ services: - "scm1.org: 172.25.0.116" - "scm2.org: 172.25.0.117" - "scm3.org: 172.25.0.118" - - "scm4.org: 172.25.0.120" + - "scm4.org: 172.25.0.220" - "recon: 172.25.0.115" env_file: - docker-config @@ -65,7 +65,7 @@ services: OZONE_OPTS: networks: ozone_net: - ipv4_address: 172.25.0.121 + ipv4_address: 172.25.0.221 profiles: [ "datanode4" ] networks: ozone_net: diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-haproxy-s3g.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh old mode 100755 new mode 100644 similarity index 84% rename from hadoop-ozone/dist/src/main/compose/ozonesecure/test-haproxy-s3g.sh rename to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh index 80470b16978d..a2b11418a88c --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-haproxy-s3g.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh @@ -15,36 +15,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -#suite:secure +#suite:HA-secure set -u -o pipefail COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR -# shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" - export SECURITY_ENABLED=true -export COMPOSE_FILE=docker-compose.yaml:../common/s3-haproxy-secure.yaml +export OM_SERVICE_ID="omservice" +export SCM=scm1.org : ${OZONE_BUCKET_KEY_NAME:=key1} +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + start_docker_env execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} -execute_robot_test scm kinit.robot - -execute_robot_test scm security - ## Exclude virtual-host tests. This is tested separately as it requires additional config. exclude="--exclude virtual-host" for bucket in encrypted; do - execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + execute_robot_test recon -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 # some tests are independent of the bucket type, only need to be run once ## Exclude virtual-host.robot exclude="--exclude virtual-host --exclude no-bucket-type" done - -execute_robot_test scm spnego diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3a.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3a.sh index b2b3c5fbda00..86e40d03c309 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3a.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3a.sh @@ -25,5 +25,6 @@ export COMPOSE_DIR export SECURITY_ENABLED=true export OM_SERVICE_ID="omservice" export SCM=scm1.org +export COMPOSE_FILE="docker-compose.yaml:s3g-port-forward.yml" source "$COMPOSE_DIR/../common/s3a-test.sh" diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh index 515b354e593a..1c134350c784 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh @@ -45,7 +45,7 @@ execute_robot_test s3g basic/links.robot ## Exclude virtual-host tests. This is tested separately as it requires additional config. exclude="--exclude virtual-host" -for bucket in encrypted link; do +for bucket in link; do execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 # some tests are independent of the bucket type, only need to be run once ## Exclude virtual-host.robot diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index 5be27b18a670..849621655905 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -253,13 +253,16 @@ execute_robot_test(){ ## @description Replace OM node order in config reorder_om_nodes() { - local c pid procname new_order + local c new_order local new_order="$1" if [[ -n "${new_order}" ]] && [[ "${new_order}" != "om1,om2,om3" ]]; then - for c in $(docker-compose ps | cut -f1 -d' ' | grep -e datanode -e recon -e s3g -e scm); do - docker exec "${c}" sed -i -e "s/om1,om2,om3/${new_order}/" /etc/hadoop/ozone-site.xml - echo "Replaced OM order with ${new_order} in ${c}" + for c in $(docker-compose ps | cut -f1 -d' ' | grep -v -e '^NAME$' -e '^om'); do + docker exec "${c}" sh -c \ + "if [[ -f /etc/hadoop/ozone-site.xml ]]; then \ + sed -i -e 's/om1,om2,om3/${new_order}/' /etc/hadoop/ozone-site.xml; \ + echo 'Replaced OM order with ${new_order} in ${c}'; \ + fi" done fi }