From 30effc27e015e74aa7da26ce11c31da951be628d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Thu, 28 Nov 2019 17:06:45 +0100 Subject: [PATCH 1/4] HDDS-2646. Start acceptance tests only if at least one THREE pipeline is available --- .../src/main/compose/ozone-hdfs/docker-config | 1 + .../src/main/compose/ozone-mr/common-config | 1 + .../main/compose/ozone-om-ha-s3/docker-config | 1 + .../main/compose/ozone-recon/docker-config | 1 + .../main/compose/ozone-topology/docker-config | 2 + .../dist/src/main/compose/ozone/docker-config | 1 + .../main/compose/ozoneblockade/docker-config | 2 + .../src/main/compose/ozoneperf/docker-config | 1 + .../compose/ozones3-haproxy/docker-config | 1 + .../src/main/compose/ozones3/docker-config | 1 + .../main/compose/ozonescripts/docker-config | 2 + .../main/compose/ozonesecure-mr/docker-config | 1 + .../main/compose/ozonesecure/docker-config | 1 + hadoop-ozone/dist/src/main/compose/testlib.sh | 49 ------------------- 14 files changed, 16 insertions(+), 49 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config index a8a96c3cbab..c6295ebe437 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config @@ -23,6 +23,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=1 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config index ee7a773cc6d..14c33ee24c8 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config @@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config index 4122daeef5d..5cfa8260391 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config @@ -29,5 +29,6 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config index bf7871e0b5a..0c817a9d99f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config @@ -27,6 +27,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 ASYNC_PROFILER_HOME=/opt/profiler diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config index ec9c59059d4..f8c939fa02c 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config @@ -29,6 +29,8 @@ OZONE-SITE.XML_ozone.scm.container.placement.impl=org.apache.hadoop.hdds.scm.con OZONE-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping OZONE-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config OZONE-SITE.XML_dfs.network.topology.aware.read.enable=true +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 + HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 ASYNC_PROFILER_HOME=/opt/profiler diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index d82cde2ea38..02b532abc65 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -25,6 +25,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 ASYNC_PROFILER_HOME=/opt/profiler diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config index 3625fc6b09a..b854aaba877 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config @@ -34,6 +34,8 @@ OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s OZONE-SITE.XML_hdds.scm.replication.thread.interval=6s OZONE-SITE.XML_hdds.scm.replication.event.timeout=10s OZONE-SITE.XML_dfs.ratis.server.failure.duration=35s +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 + HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config index 931f112508d..5d68afc28e8 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config @@ -25,6 +25,7 @@ OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 ASYNC_PROFILER_HOME=/opt/profiler diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config index 75d197759b2..4c08c52bd1a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config @@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config index 75d197759b2..4c08c52bd1a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config @@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config index 99508981379..787ef2efd5b 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config @@ -26,6 +26,8 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=1 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 + HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode HDFS-SITE.XML_rpc.metrics.quantile.enable=true diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index 7d6fdc4dd0f..54992d83af7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -31,6 +31,7 @@ OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab OZONE-SITE.XML_ozone.s3g.keytab.file=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_ozone.s3g.authentication.kerberos.principal=HTTP/s3g@EXAMPLE.COM OZONE-SITE.XML_ozone.administrators=* +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_ozone.security.enabled=true OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index ff422491092..14ef63cf7f6 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -36,6 +36,7 @@ OZONE-SITE.XML_ozone.recon.authentication.kerberos.principal=HTTP/recon@EXAMPLE. OZONE-SITE.XML_ozone.recon.keytab.file=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_recon.om.snapshot.task.initial.delay=20s +OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_ozone.security.enabled=true OZONE-SITE.XML_ozone.acl.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index 49274c0dfb2..f4b0fbca239 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -31,54 +31,6 @@ create_results_dir() { chmod ogu+w "$RESULT_DIR" } -## @description print the number of datanodes up -## @param the docker-compose file -count_datanodes() { - local compose_file=$1 - - local jmx_url='http://scm:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' - if [[ "${SECURITY_ENABLED}" == 'true' ]]; then - docker-compose -f "${compose_file}" exec -T scm bash -c "kinit -k HTTP/scm@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab && curl --negotiate -u : -s '${jmx_url}'" - else - docker-compose -f "${compose_file}" exec -T scm curl -s "${jmx_url}" - fi \ - | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value' || true -} - -## @description wait until datanodes are up (or 30 seconds) -## @param the docker-compose file -## @param number of datanodes to wait for (default: 3) -wait_for_datanodes(){ - local compose_file=$1 - local -i datanode_count=${2:-3} - - #Reset the timer - SECONDS=0 - - #Don't give it up until 30 seconds - while [[ $SECONDS -lt 90 ]]; do - - #This line checks the number of HEALTHY datanodes registered in scm over the - # jmx HTTP servlet - datanodes=$(count_datanodes "${compose_file}") - if [[ "$datanodes" ]]; then - if [[ ${datanodes} -ge ${datanode_count} ]]; then - - #It's up and running. Let's return from the function. - echo "$datanodes datanodes are up and registered to the scm" - return - else - - #Print it only if a number. Could be not a number if scm is not yet started - echo "$datanodes datanode is up and healthy (until now)" - fi - fi - - sleep 2 - done - echo "WARNING! Datanodes are not started successfully. Please check the docker-compose files" - return 1 -} ## @description wait until safemode exit (or 30 seconds) ## @param the docker-compose file @@ -123,7 +75,6 @@ start_docker_env(){ docker-compose -f "$COMPOSE_FILE" --no-ansi down docker-compose -f "$COMPOSE_FILE" --no-ansi up -d --scale datanode="${datanode_count}" \ - && wait_for_datanodes "$COMPOSE_FILE" "${datanode_count}" \ && wait_for_safemode_exit "$COMPOSE_FILE" \ && sleep 10 From 41733fc297aeff0fb2003c6913ff4c51ff1ec927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Thu, 28 Nov 2019 22:44:22 +0100 Subject: [PATCH 2/4] retrigger build From bd99167e4fbda501568361776014237d17bdbe8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Tue, 3 Dec 2019 12:28:31 +0100 Subject: [PATCH 3/4] remove configs where we don't need them --- hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config | 1 - hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config index c6295ebe437..a8a96c3cbab 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config @@ -23,7 +23,6 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=1 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config index 787ef2efd5b..7f414621d2a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config @@ -26,10 +26,9 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=1 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService \ No newline at end of file +HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService From 1300a0aac8ec6ec6a7f3ae83ca5e3aeffd7da8b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Tue, 3 Dec 2019 12:54:22 +0100 Subject: [PATCH 4/4] switch to use ENV based variable with default --- .../dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml | 1 + .../dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml | 1 + .../dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml | 1 + .../dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config | 1 - .../dist/src/main/compose/ozone-recon/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config | 1 - .../dist/src/main/compose/ozone-topology/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config | 1 - hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozone/docker-config | 1 - .../dist/src/main/compose/ozoneperf/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config | 1 - .../dist/src/main/compose/ozones3-haproxy/docker-compose.yaml | 1 + .../dist/src/main/compose/ozones3-haproxy/docker-config | 1 - hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozones3/docker-config | 1 - .../dist/src/main/compose/ozonesecure-mr/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config | 1 - .../dist/src/main/compose/ozonesecure/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config | 1 - hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +- 22 files changed, 13 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml index 17f5ee53552..85f75b0e9a9 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml @@ -63,6 +63,7 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml index e3696fcf70a..9a30c8e32a5 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml @@ -63,6 +63,7 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml index c25d36cb904..3462a91b42c 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml @@ -63,6 +63,7 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml index 390acb37acb..ff7f260c44e 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml @@ -76,6 +76,7 @@ services: - ./docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] s3g: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config index 5cfa8260391..4122daeef5d 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config @@ -29,6 +29,5 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml index 38e2ef33094..3f010a2d79a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml @@ -50,6 +50,7 @@ services: - ./docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] recon: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config index 0c817a9d99f..bf7871e0b5a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config @@ -27,7 +27,6 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 ASYNC_PROFILER_HOME=/opt/profiler diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml index a66eff617ed..69611fa674c 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml @@ -98,6 +98,7 @@ services: - ./docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] networks: net: diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config index f8c939fa02c..611c6c004a9 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config @@ -29,7 +29,6 @@ OZONE-SITE.XML_ozone.scm.container.placement.impl=org.apache.hadoop.hdds.scm.con OZONE-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping OZONE-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config OZONE-SITE.XML_dfs.network.topology.aware.read.enable=true -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index 145ce3ebb17..6dc9d14ac5b 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -50,4 +50,5 @@ services: - ./docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index 02b532abc65..d82cde2ea38 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -25,7 +25,6 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 ASYNC_PROFILER_HOME=/opt/profiler diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml index 3c6618da68a..b78cf098df3 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml @@ -49,6 +49,7 @@ services: - ./docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["ozone","scm"] jaeger: image: jaegertracing/all-in-one:latest diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config index 5d68afc28e8..931f112508d 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config @@ -25,7 +25,6 @@ OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 ASYNC_PROFILER_HOME=/opt/profiler diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml index 78fd996a70c..25ed6f6c923 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml @@ -53,6 +53,7 @@ services: - ./docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["ozone","scm"] s3g1: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config index 4c08c52bd1a..75d197759b2 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config @@ -22,7 +22,6 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml index cc4bfd2268c..26bcee5820a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml @@ -46,6 +46,7 @@ services: - ./docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["ozone","scm"] s3g: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config index 4c08c52bd1a..75d197759b2 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config @@ -22,7 +22,6 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml index 53e0142b2b6..1b8fc9c668f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml @@ -86,6 +86,7 @@ services: - docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] rm: image: apache/hadoop:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index 54992d83af7..7d6fdc4dd0f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -31,7 +31,6 @@ OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab OZONE-SITE.XML_ozone.s3g.keytab.file=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_ozone.s3g.authentication.kerberos.principal=HTTP/s3g@EXAMPLE.COM OZONE-SITE.XML_ozone.administrators=* -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_ozone.security.enabled=true OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml index de60a411116..65f24f963c1 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml @@ -88,4 +88,5 @@ services: - docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" command: ["/opt/hadoop/bin/ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index 14ef63cf7f6..ff422491092 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -36,7 +36,6 @@ OZONE-SITE.XML_ozone.recon.authentication.kerberos.principal=HTTP/recon@EXAMPLE. OZONE-SITE.XML_ozone.recon.keytab.file=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_recon.om.snapshot.task.initial.delay=20s -OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_ozone.security.enabled=true OZONE-SITE.XML_ozone.acl.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index f4b0fbca239..4375c3dd2d5 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -72,7 +72,7 @@ start_docker_env(){ local -i datanode_count=${1:-3} create_results_dir - + export OZONE_SAFEMODE_MIN_DATANODE="${datanode_count}" docker-compose -f "$COMPOSE_FILE" --no-ansi down docker-compose -f "$COMPOSE_FILE" --no-ansi up -d --scale datanode="${datanode_count}" \ && wait_for_safemode_exit "$COMPOSE_FILE" \