From 800cda254f50bf403eb0aa25ac5a64d626c54367 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 20 Nov 2019 16:26:11 +0100 Subject: [PATCH 1/6] HDDS-2588. Consolidate compose environments --- .../dist/src/main/compose/ozone-recon/.env | 18 ----- .../compose/ozone-recon/docker-compose.yaml | 65 ------------------- .../main/compose/ozone-recon/docker-config | 35 ---------- .../dist/src/main/compose/ozone-recon/test.sh | 30 --------- hadoop-ozone/dist/src/main/compose/ozone/.env | 18 ----- .../main/compose/ozone/docker-compose.yaml | 53 --------------- .../dist/src/main/compose/ozone/docker-config | 33 ---------- .../dist/src/main/compose/ozone/test.sh | 38 ----------- .../compose/ozoneperf/docker-compose.yaml | 19 +++++- .../src/main/compose/ozoneperf/docker-config | 5 ++ .../dist/src/main/compose/ozoneperf/test.sh | 14 +++- .../dist/src/main/compose/ozones3/.env | 18 ----- .../main/compose/ozones3/docker-compose.yaml | 58 ----------------- .../src/main/compose/ozones3/docker-config | 30 --------- .../dist/src/main/compose/ozones3/test.sh | 34 ---------- 15 files changed, 34 insertions(+), 434 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-recon/.env delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config delete mode 100755 hadoop-ozone/dist/src/main/compose/ozone-recon/test.sh delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/.env delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/docker-config delete mode 100755 hadoop-ozone/dist/src/main/compose/ozone/test.sh delete mode 100644 hadoop-ozone/dist/src/main/compose/ozones3/.env delete mode 100644 hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml delete mode 100644 hadoop-ozone/dist/src/main/compose/ozones3/docker-config delete mode 100755 hadoop-ozone/dist/src/main/compose/ozones3/test.sh diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/.env b/hadoop-ozone/dist/src/main/compose/ozone-recon/.env deleted file mode 100644 index 96ab163b4747..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/.env +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} -OZONE_RUNNER_VERSION=${docker.ozone-runner.version} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml deleted file mode 100644 index 38e2ef330940..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - - 9882 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] - scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] - recon: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9888:9888 - env_file: - - ./docker-config - environment: - WAITFOR: om:9874 - command: ["/opt/hadoop/bin/ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config deleted file mode 100644 index bf7871e0b5a9..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon -OZONE-SITE.XML_ozone.recon.om.db.dir=/data/metadata/recon -OZONE-SITE.XML_ozone.handler.type=distributed -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.replication=3 -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -ASYNC_PROFILER_HOME=/opt/profiler - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-recon/test.sh deleted file mode 100755 index f4bfcc3d57f9..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/test.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export COMPOSE_DIR - -# shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" - -start_docker_env - -execute_robot_test scm basic/basic.robot - -stop_docker_env - -generate_report diff --git a/hadoop-ozone/dist/src/main/compose/ozone/.env b/hadoop-ozone/dist/src/main/compose/ozone/.env deleted file mode 100644 index 96ab163b4747..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/.env +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} -OZONE_RUNNER_VERSION=${docker.ozone-runner.version} diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml deleted file mode 100644 index 145ce3ebb176..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - - 9882 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] - scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config deleted file mode 100644 index d82cde2ea386..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.handler.type=distributed -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.replication=3 -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -ASYNC_PROFILER_HOME=/opt/profiler - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh deleted file mode 100755 index e06f817f3d81..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export COMPOSE_DIR - -# shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" - -start_docker_env - -#Due to the limitation of the current auditparser test, it should be the -#first test in a clean cluster. - -#Disabling for now, audit parser tool during parse getting exception. -#execute_robot_test om auditparser - -execute_robot_test scm basic/basic.robot - -execute_robot_test scm gdpr/gdpr.robot - -stop_docker_env - -generate_report diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml index 3c6618da68a1..c9bc46a4547f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml @@ -18,11 +18,12 @@ version: "3" services: datanode: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true + privileged: true # required by the profiler volumes: - ../..:/opt/hadoop ports: - 9864 + - 9882 command: ["ozone","datanode"] env_file: - ./docker-config @@ -40,7 +41,7 @@ services: command: ["ozone","om"] scm: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true + privileged: true # required by the profiler volumes: - ../..:/opt/hadoop ports: @@ -74,7 +75,7 @@ services: - 3000:3000 s3g: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true + privileged: true # required by the profiler volumes: - ../..:/opt/hadoop ports: @@ -82,3 +83,15 @@ services: env_file: - ./docker-config command: ["ozone","s3g"] + recon: + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + privileged: true # required by the profiler + volumes: + - ../..:/opt/hadoop + ports: + - 9888:9888 + env_file: + - ./docker-config + environment: + WAITFOR: om:9874 + command: ["ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config index 931f112508d2..919b0e49e141 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config @@ -20,6 +20,8 @@ OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon +OZONE-SITE.XML_ozone.recon.om.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_ozone.replication=3 @@ -34,3 +36,6 @@ HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 JAEGER_SAMPLER_PARAM=1 JAEGER_SAMPLER_TYPE=const JAEGER_AGENT_HOST=jaeger + +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh b/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh index f4bfcc3d57f9..4284910ce784 100755 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh @@ -18,12 +18,24 @@ COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR +export SECURITY_ENABLED=false + # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" start_docker_env -execute_robot_test scm basic/basic.robot +#Due to the limitation of the current auditparser test, it should be the +#first test in a clean cluster. + +#Disabling for now, audit parser tool during parse getting exception. +#execute_robot_test om auditparser + +execute_robot_test scm basic + +execute_robot_test scm gdpr + +execute_robot_test scm s3 stop_docker_env diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/.env b/hadoop-ozone/dist/src/main/compose/ozones3/.env deleted file mode 100644 index 96ab163b4747..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozones3/.env +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} -OZONE_RUNNER_VERSION=${docker.ozone-runner.version} diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml deleted file mode 100644 index cc4bfd2268c6..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - command: ["ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["ozone","om"] - scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["ozone","scm"] - s3g: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9878:9878 - env_file: - - ./docker-config - command: ["ozone","s3g"] diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config deleted file mode 100644 index 75d197759b2e..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.replication=3 -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds - -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh b/hadoop-ozone/dist/src/main/compose/ozones3/test.sh deleted file mode 100755 index a534a627f201..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export COMPOSE_DIR - -export SECURITY_ENABLED=false - -# shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" - -start_docker_env - -execute_robot_test scm basic/basic.robot - -execute_robot_test scm s3 - -stop_docker_env - -generate_report From b4b586602373fafd13e32ee9815e1f7bf3c6ef41 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 20 Nov 2019 16:28:30 +0100 Subject: [PATCH 2/6] HDDS-2588. Rename ozoneperf compose env to ozone --- .../main/compose/{ozoneperf => ozone}/.env | 0 .../compose/{ozoneperf => ozone}/README.md | 0 .../{ozoneperf => ozone}/docker-compose.yaml | 0 .../{ozoneperf => ozone}/docker-config | 0 .../{ozoneperf => ozone}/freon-ockg.yaml | 0 .../{ozoneperf => ozone}/freon-rk.yaml | 0 .../{ozoneperf => ozone}/prometheus.yml | 20 +++++++++---------- .../main/compose/{ozoneperf => ozone}/test.sh | 0 8 files changed, 10 insertions(+), 10 deletions(-) rename hadoop-ozone/dist/src/main/compose/{ozoneperf => ozone}/.env (100%) rename hadoop-ozone/dist/src/main/compose/{ozoneperf => ozone}/README.md (100%) rename hadoop-ozone/dist/src/main/compose/{ozoneperf => ozone}/docker-compose.yaml (100%) rename hadoop-ozone/dist/src/main/compose/{ozoneperf => ozone}/docker-config (100%) rename hadoop-ozone/dist/src/main/compose/{ozoneperf => ozone}/freon-ockg.yaml (100%) rename hadoop-ozone/dist/src/main/compose/{ozoneperf => ozone}/freon-rk.yaml (100%) rename hadoop-ozone/dist/src/main/compose/{ozoneperf => ozone}/prometheus.yml (72%) rename hadoop-ozone/dist/src/main/compose/{ozoneperf => ozone}/test.sh (100%) diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/.env b/hadoop-ozone/dist/src/main/compose/ozone/.env similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/.env rename to hadoop-ozone/dist/src/main/compose/ozone/.env diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md b/hadoop-ozone/dist/src/main/compose/ozone/README.md similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/README.md rename to hadoop-ozone/dist/src/main/compose/ozone/README.md diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml rename to hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config rename to hadoop-ozone/dist/src/main/compose/ozone/docker-config diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/freon-ockg.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/freon-ockg.yaml rename to hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/freon-rk.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/freon-rk.yaml rename to hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml similarity index 72% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml rename to hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml index d4f475b0ff14..48e6ac5c6605 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml +++ b/hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml @@ -23,13 +23,13 @@ scrape_configs: - targets: - "scm:9876" - "om:9874" - - "ozoneperf_datanode_1:9882" - - "ozoneperf_datanode_2:9882" - - "ozoneperf_datanode_3:9882" - - "ozoneperf_datanode_4:9882" - - "ozoneperf_datanode_5:9882" - - "ozoneperf_datanode_6:9882" - - "ozoneperf_datanode_7:9882" - - "ozoneperf_datanode_8:9882" - - "ozoneperf_datanode_9:9882" - - "ozoneperf_datanode_10:9882" + - "ozone_datanode_1:9882" + - "ozone_datanode_2:9882" + - "ozone_datanode_3:9882" + - "ozone_datanode_4:9882" + - "ozone_datanode_5:9882" + - "ozone_datanode_6:9882" + - "ozone_datanode_7:9882" + - "ozone_datanode_8:9882" + - "ozone_datanode_9:9882" + - "ozone_datanode_10:9882" diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh rename to hadoop-ozone/dist/src/main/compose/ozone/test.sh From 16771752912e5d05fb704e5409822604654f23ca Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 21 Nov 2019 20:31:27 +0100 Subject: [PATCH 3/6] HDDS-2588. Extract monitoring and profiling into composable parts --- .../main/compose/ozone/docker-compose.yaml | 123 ++++++------------ .../dist/src/main/compose/ozone/docker-config | 14 -- .../src/main/compose/ozone/monitoring.yaml | 63 +++++++++ .../src/main/compose/ozone/profiling.yaml | 36 +++++ 4 files changed, 142 insertions(+), 94 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index c9bc46a4547f..6c5167d47e38 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -14,84 +14,47 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" + +# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) +x-common-config: + &common-config + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + services: - datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true # required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - - 9882 - command: ["ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["ozone","om"] - scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true # required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["ozone","scm"] - jaeger: - image: jaegertracing/all-in-one:latest - environment: - COLLECTOR_ZIPKIN_HTTP_PORT: 9411 - ports: - - 16686:16686 - prometheus: - image: prom/prometheus - volumes: - - "./prometheus.yml:/etc/prometheus.yml" - command: ["--config.file","/etc/prometheus.yml"] - ports: - - 9090:9090 - grafana: - image: grafana/grafana - volumes: - - "../common/grafana/dashboards:/var/lib/grafana/dashboards" - - "../common/grafana/provisioning:/etc/grafana/provisioning" - - "../common/grafana/conf/grafana.ini:/etc/grafana/grafana.ini" - command: ["-config","/etc/grafana/grafana.ini"] - ports: - - 3000:3000 - s3g: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true # required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9878:9878 - env_file: - - ./docker-config - command: ["ozone","s3g"] - recon: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true # required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9888:9888 - env_file: - - ./docker-config - environment: - WAITFOR: om:9874 - command: ["ozone","recon"] + datanode: + <<: *common-config + ports: + - 9864 + - 9882 + command: ["ozone","datanode"] + om: + <<: *common-config + ports: + - 9874:9874 + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + command: ["ozone","om"] + scm: + <<: *common-config + ports: + - 9876:9876 + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["ozone","scm"] + s3g: + <<: *common-config + ports: + - 9878:9878 + command: ["ozone","s3g"] + recon: + <<: *common-config + ports: + - 9888:9888 + environment: + WAITFOR: om:9874 + command: ["ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index 919b0e49e141..7b1c82a4ea82 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -25,17 +25,3 @@ OZONE-SITE.XML_ozone.recon.om.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_ozone.replication=3 -OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true -OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true - -ASYNC_PROFILER_HOME=/opt/profiler - -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 - -JAEGER_SAMPLER_PARAM=1 -JAEGER_SAMPLER_TYPE=const -JAEGER_AGENT_HOST=jaeger - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml new file mode 100644 index 000000000000..47b0f3d2a739 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.4" + +x-monitoring-config: + &monitoring-config + environment: + - OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true + - HDFS-SITE.XML_rpc.metrics.quantile.enable=true + - HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 + - JAEGER_SAMPLER_PARAM=1 + - JAEGER_SAMPLER_TYPE=const + - JAEGER_AGENT_HOST=jaeger + #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. + #- BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + +services: + jaeger: + image: jaegertracing/all-in-one:latest + environment: + COLLECTOR_ZIPKIN_HTTP_PORT: 9411 + ports: + - 16686:16686 + prometheus: + image: prom/prometheus + volumes: + - "./prometheus.yml:/etc/prometheus.yml" + command: ["--config.file","/etc/prometheus.yml"] + ports: + - 9090:9090 + grafana: + image: grafana/grafana + volumes: + - "../common/grafana/dashboards:/var/lib/grafana/dashboards" + - "../common/grafana/provisioning:/etc/grafana/provisioning" + - "../common/grafana/conf/grafana.ini:/etc/grafana/grafana.ini" + command: ["-config","/etc/grafana/grafana.ini"] + ports: + - 3000:3000 + datanode: + <<: *monitoring-config + om: + <<: *monitoring-config + scm: + <<: *monitoring-config + s3g: + <<: *monitoring-config + recon: + <<: *monitoring-config diff --git a/hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml b/hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml new file mode 100644 index 000000000000..f3f14f6288c8 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.4" + +x-profiling-config: + &profiling-config + privileged: true + environment: + - OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true + - ASYNC_PROFILER_HOME=/opt/profiler + +services: + datanode: + <<: *profiling-config + om: + <<: *profiling-config + scm: + <<: *profiling-config + s3g: + <<: *profiling-config + recon: + <<: *profiling-config From 246e35bd15fbe0ce8c065221abdb12b0e34bb09d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sat, 30 Nov 2019 22:21:17 +0100 Subject: [PATCH 4/6] HDDS-2588. Update README, fix Freon --- .../dist/src/main/compose/ozone/README.md | 59 ++++++++++--------- .../src/main/compose/ozone/freon-ockg.yaml | 17 +++--- .../dist/src/main/compose/ozone/freon-rk.yaml | 17 +++--- .../src/main/compose/ozone/monitoring.conf | 25 ++++++++ .../src/main/compose/ozone/monitoring.yaml | 11 +--- 5 files changed, 77 insertions(+), 52 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf diff --git a/hadoop-ozone/dist/src/main/compose/ozone/README.md b/hadoop-ozone/dist/src/main/compose/ozone/README.md index d8aa8c3dcaeb..ddcb190811a7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/README.md +++ b/hadoop-ozone/dist/src/main/compose/ozone/README.md @@ -12,43 +12,51 @@ limitations under the License. See accompanying LICENSE file. --> -# Compose files for local performance tests +# Compose file with optional monitoring and profiling configs -This directory contains docker-compose definition for an ozone cluster where -all the metrics are saved to a prometheus instance, and profiling and Jaeger -tracing is turned on and set up. +This directory contains a docker-compose definition for an Ozone cluster with all components (including S3 Gateway and Recon). -Prometheus follows a pull based approach where the metrics are published - on a HTTP endpoint. +There are two optional add-ons: -Prometheus compatible metrics endpoint can be enabled by setting `hdds.prometheus.endpoint.enabled` property to `true` + * monitoring: adds Grafana, Jaeger and Prometheus sercvies, and configures Ozone to work with them + * profiling: allows sampling Ozone CPU/memory using [async-profiler](https://github.com/jvm-profiling-tools/async-profiler) ## How to start Start the cluster with `docker-compose` -``` -docker-compose up -d -``` + * one datanode: `docker-compose up -d` + * three datanodes: `docker-compose up -d --scale datanode=3` + +### Add-ons -Scale datanodes up: +Monitoring and/or performance add-ons can be enabled via docker-compose's ability to use multiple compose files (by using the [`-f` option repeatedly](https://docs.docker.com/compose/reference/overview/#specifying-multiple-compose-files), or more easily by defining the [`COMPOSE_FILE` environment variable](https://docs.docker.com/compose/reference/envvars/#compose_file)): ``` -docker-compose up -d --scale datanode=3 +# no COMPOSE_FILE var # => only Ozone +export COMPOSE_FILE=docker-compose.yaml:monitoring.yaml # => add monitoring +export COMPOSE_FILE=docker-compose.yaml:profiling.yaml # => add profiling +export COMPOSE_FILE=docker-compose.yaml:monitoring.yaml:profiling.yaml # => add both ``` -You can enter to the SCM container and start a freon test: +Once the variable is defined, Ozone cluster with add-ons can be started/scaled/stopped etc. using the same `docker-compose` commands as for the base cluster. + +### Load generator + +Ozone comes with a load generator called Freon. + +You can enter one of the containers (eg. SCM) and start a Freon test: ``` docker-compose exec scm bash ozone freon ockg -n1000 ``` -Or you can start freon instances in containers (to make it possible to scale them up): -If all the datanodes are started, start the freon instance: +You can also start two flavors of Freon as separate services, which allows scaling it up. Once all the datanodes are started, start Freon by adding its definition to `COMPOSE_FILE` and re-running the `docker-compose up` command: ``` -docker-compose -f docker-compose.yaml -f freon-ockg.yaml up --scale datanode=3 -d +export COMPOSE_FILE="${COMPOSE_FILE}:freon-ockg.yaml" +docker-compose up -d --scale datanode=3 ``` ## How to use @@ -58,17 +66,14 @@ You can check the ozone web ui: OzoneManager: http://localhost:9874 SCM: http://localhost:9876 -You can check the ozone metrics from the prometheus web ui. - -http://localhost:9090/graph - -You can view Grafana dashboards at: +### Monitoring -http://localhost:3000 + * Prometheus: follows a pull based approach where metrics are published on an HTTP endpoint. Metrics can be checked on [Prometheus' web UI](http://localhost:9090/) + * Grafana: comes with two [dashboards](http://localhost:3000) for Ozone + * Ozone - Object Metrics + * Ozone - RPC Metrics + * Jaeger: collects distributed tracing information from Ozone, can be queried on the [Jaeger web UI](http://localhost:16686) -Default dashboards available are: -Ozone - Object Metrics -Ozone - RPC Metrics +### Profiling -You can access the Jaeger UI at: -http://localhost:16686 \ No newline at end of file +Start by hitting the `/prof` endpoint on the service to be profiled, eg. http://localhost:9876/prof for SCM. [Detailed instructions](https://cwiki.apache.org/confluence/display/HADOOP/Java+Profiling+of+Ozone) can be found in the Hadoop wiki. diff --git a/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml index 72685bdc6727..8cf6ee70e978 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml @@ -14,12 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" services: - freon: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - ./docker-config - command: ["ozone","freon","ockg","-n100000"] + freon: + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + - monitoring.conf + command: ["ozone","freon","ockg","-n100000"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml index 671bd229a328..fbce3ad649f3 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml @@ -14,12 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" services: - freon: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - ./docker-config - command: ["ozone","freon","rk"] + freon: + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + - monitoring.conf + command: ["ozone","freon","rk"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf new file mode 100644 index 000000000000..156d539d6178 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true +OZONE-SITE.XML_hdds.tracing.enabled=true +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +JAEGER_SAMPLER_PARAM=1 +JAEGER_SAMPLER_TYPE=const +JAEGER_AGENT_HOST=jaeger +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml index 47b0f3d2a739..e08cc8ec5a05 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml @@ -18,15 +18,8 @@ version: "3.4" x-monitoring-config: &monitoring-config - environment: - - OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true - - HDFS-SITE.XML_rpc.metrics.quantile.enable=true - - HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 - - JAEGER_SAMPLER_PARAM=1 - - JAEGER_SAMPLER_TYPE=const - - JAEGER_AGENT_HOST=jaeger - #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. - #- BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + env_file: + - monitoring.conf services: jaeger: From 382e2a5d29465334e358b62b2f2dd9311e0b8406 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 1 Dec 2019 13:32:39 +0100 Subject: [PATCH 5/6] HDDS-2588. Add run.sh for convenience; control replication factor via env. variable --- .../dist/src/main/compose/ozone/README.md | 30 +++++++++++++++---- .../main/compose/ozone/docker-compose.yaml | 18 +++++++---- .../dist/src/main/compose/ozone/docker-config | 1 - .../src/main/compose/ozone/freon-ockg.yaml | 3 ++ .../dist/src/main/compose/ozone/freon-rk.yaml | 3 ++ .../dist/src/main/compose/ozone/om.conf | 17 +++++++++++ .../dist/src/main/compose/ozone/recon.conf | 17 +++++++++++ .../dist/src/main/compose/ozone/run.sh | 20 +++++++++++++ .../dist/src/main/compose/ozone/scm.conf | 17 +++++++++++ .../dist/src/main/compose/ozone/test.sh | 1 + 10 files changed, 115 insertions(+), 12 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/om.conf create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/recon.conf create mode 100755 hadoop-ozone/dist/src/main/compose/ozone/run.sh create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/scm.conf diff --git a/hadoop-ozone/dist/src/main/compose/ozone/README.md b/hadoop-ozone/dist/src/main/compose/ozone/README.md index ddcb190811a7..c28f83247050 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/README.md +++ b/hadoop-ozone/dist/src/main/compose/ozone/README.md @@ -23,10 +23,27 @@ There are two optional add-ons: ## How to start -Start the cluster with `docker-compose` +TL;DR: - * one datanode: `docker-compose up -d` - * three datanodes: `docker-compose up -d --scale datanode=3` +1. single datanode: + ``` + ./run.sh -d + ``` +2. three datanodes for replication: + ``` + export OZONE_REPLICATION_FACTOR=3 + ./run.sh -d + ``` + +### Basics + +The cluster can be started with regular `docker-compose up` command. Use `-d` to start the cluster in the background. + +You can change the number of datanodes to start using the `--scale` option. Eg. to start 3 datanodes: `docker-compose up -d --scale datanode=3`. + +The cluster's replication factor (1 or 3) can be controlled by setting the `OZONE_REPLICATION_FACTOR` environment variable. It defaults to 1 to match the number of datanodes started by default, without the `--scale` option. + +For convenience the `run.sh` script can be used to make sure the replication factor and the number of datanodes match. It also passes any additional arguments provided on the command-line (eg. `-d`) to `docker-compose`. ### Add-ons @@ -52,11 +69,14 @@ docker-compose exec scm bash ozone freon ockg -n1000 ``` -You can also start two flavors of Freon as separate services, which allows scaling it up. Once all the datanodes are started, start Freon by adding its definition to `COMPOSE_FILE` and re-running the `docker-compose up` command: +You can also start two flavors of Freon as separate services, which allows scaling them up. Once all the datanodes are started, start Freon by adding its definition to `COMPOSE_FILE` and re-running the `docker-compose up` or `run.sh` command: ``` export COMPOSE_FILE="${COMPOSE_FILE}:freon-ockg.yaml" -docker-compose up -d --scale datanode=3 + +docker-compose up -d --no-recreate --scale datanode=3 +# OR +./run.sh -d ``` ## How to use diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index 6c5167d47e38..8395617efd76 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -24,6 +24,9 @@ x-common-config: - ../..:/opt/hadoop env_file: - docker-config + environment: + - "OZONE-SITE.XML_hdds.scm.safemode.min.datanode=${OZONE_REPLICATION_FACTOR:-1}" + - "OZONE-SITE.XML_ozone.replication=${OZONE_REPLICATION_FACTOR:-1}" services: datanode: @@ -34,17 +37,19 @@ services: command: ["ozone","datanode"] om: <<: *common-config + env_file: + - docker-config + - om.conf ports: - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION command: ["ozone","om"] scm: <<: *common-config ports: - 9876:9876 - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + env_file: + - docker-config + - scm.conf command: ["ozone","scm"] s3g: <<: *common-config @@ -55,6 +60,7 @@ services: <<: *common-config ports: - 9888:9888 - environment: - WAITFOR: om:9874 + env_file: + - docker-config + - recon.conf command: ["ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index 7b1c82a4ea82..c4d79c14dfac 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -24,4 +24,3 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.recon.om.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_ozone.replication=3 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml index 8cf6ee70e978..afde1009dd49 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml @@ -23,4 +23,7 @@ services: env_file: - docker-config - monitoring.conf + environment: + - "OZONE-SITE.XML_hdds.scm.safemode.min.datanode=${OZONE_REPLICATION_FACTOR:-1}" + - "OZONE-SITE.XML_ozone.replication=${OZONE_REPLICATION_FACTOR:-1}" command: ["ozone","freon","ockg","-n100000"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml index fbce3ad649f3..0884bc575c97 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml @@ -23,4 +23,7 @@ services: env_file: - docker-config - monitoring.conf + environment: + - "OZONE-SITE.XML_hdds.scm.safemode.min.datanode=${OZONE_REPLICATION_FACTOR:-1}" + - "OZONE-SITE.XML_ozone.replication=${OZONE_REPLICATION_FACTOR:-1}" command: ["ozone","freon","rk"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/om.conf b/hadoop-ozone/dist/src/main/compose/ozone/om.conf new file mode 100644 index 000000000000..a59dacbb2553 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/om.conf @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ENSURE_OM_INITIALIZED=/data/metadata/om/current/VERSION diff --git a/hadoop-ozone/dist/src/main/compose/ozone/recon.conf b/hadoop-ozone/dist/src/main/compose/ozone/recon.conf new file mode 100644 index 000000000000..4ee361690880 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/recon.conf @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +WAITFOR=om:9874 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/run.sh b/hadoop-ozone/dist/src/main/compose/ozone/run.sh new file mode 100755 index 000000000000..027d69722fcc --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/run.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +declare -ix OZONE_REPLICATION_FACTOR +: ${OZONE_REPLICATION_FACTOR:=1} +docker-compose up --scale datanode=${OZONE_REPLICATION_FACTOR} --no-recreate "$@" diff --git a/hadoop-ozone/dist/src/main/compose/ozone/scm.conf b/hadoop-ozone/dist/src/main/compose/ozone/scm.conf new file mode 100644 index 000000000000..4536a3f08b90 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/scm.conf @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ENSURE_SCM_INITIALIZED=/data/metadata/scm/current/VERSION diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index 4284910ce784..7decbd11e62c 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -19,6 +19,7 @@ COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR export SECURITY_ENABLED=false +export OZONE_REPLICATION_FACTOR=3 # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" From ae2bc55261fff4950ac6cb82f7dfab1cd8e47034 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 4 Dec 2019 05:45:24 +0100 Subject: [PATCH 6/6] HDDS-2588. No need for one-liner config files --- .../main/compose/ozone/docker-compose.yaml | 30 +++++++++++-------- .../src/main/compose/ozone/freon-ockg.yaml | 1 - .../dist/src/main/compose/ozone/freon-rk.yaml | 1 - .../dist/src/main/compose/ozone/om.conf | 17 ----------- .../dist/src/main/compose/ozone/recon.conf | 17 ----------- .../dist/src/main/compose/ozone/scm.conf | 17 ----------- 6 files changed, 18 insertions(+), 65 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/om.conf delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/recon.conf delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/scm.conf diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index 8395617efd76..ffc2bce0e362 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -24,9 +24,11 @@ x-common-config: - ../..:/opt/hadoop env_file: - docker-config - environment: - - "OZONE-SITE.XML_hdds.scm.safemode.min.datanode=${OZONE_REPLICATION_FACTOR:-1}" - - "OZONE-SITE.XML_ozone.replication=${OZONE_REPLICATION_FACTOR:-1}" + +x-replication: + &replication + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_REPLICATION_FACTOR:-1} + OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1} services: datanode: @@ -34,12 +36,14 @@ services: ports: - 9864 - 9882 + environment: + <<: *replication command: ["ozone","datanode"] om: <<: *common-config - env_file: - - docker-config - - om.conf + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication ports: - 9874:9874 command: ["ozone","om"] @@ -47,12 +51,14 @@ services: <<: *common-config ports: - 9876:9876 - env_file: - - docker-config - - scm.conf + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + <<: *replication command: ["ozone","scm"] s3g: <<: *common-config + environment: + <<: *replication ports: - 9878:9878 command: ["ozone","s3g"] @@ -60,7 +66,7 @@ services: <<: *common-config ports: - 9888:9888 - env_file: - - docker-config - - recon.conf + environment: + WAITFOR: om:9874 + <<: *replication command: ["ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml index afde1009dd49..765827d44b75 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml @@ -24,6 +24,5 @@ services: - docker-config - monitoring.conf environment: - - "OZONE-SITE.XML_hdds.scm.safemode.min.datanode=${OZONE_REPLICATION_FACTOR:-1}" - "OZONE-SITE.XML_ozone.replication=${OZONE_REPLICATION_FACTOR:-1}" command: ["ozone","freon","ockg","-n100000"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml index 0884bc575c97..7bb1dbf4b1f8 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml @@ -24,6 +24,5 @@ services: - docker-config - monitoring.conf environment: - - "OZONE-SITE.XML_hdds.scm.safemode.min.datanode=${OZONE_REPLICATION_FACTOR:-1}" - "OZONE-SITE.XML_ozone.replication=${OZONE_REPLICATION_FACTOR:-1}" command: ["ozone","freon","rk"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/om.conf b/hadoop-ozone/dist/src/main/compose/ozone/om.conf deleted file mode 100644 index a59dacbb2553..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/om.conf +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ENSURE_OM_INITIALIZED=/data/metadata/om/current/VERSION diff --git a/hadoop-ozone/dist/src/main/compose/ozone/recon.conf b/hadoop-ozone/dist/src/main/compose/ozone/recon.conf deleted file mode 100644 index 4ee361690880..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/recon.conf +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -WAITFOR=om:9874 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/scm.conf b/hadoop-ozone/dist/src/main/compose/ozone/scm.conf deleted file mode 100644 index 4536a3f08b90..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone/scm.conf +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ENSURE_SCM_INITIALIZED=/data/metadata/scm/current/VERSION