diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/.env b/hadoop-ozone/dist/src/main/compose/ozone-recon/.env deleted file mode 100644 index 96ab163b474..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/.env +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} -OZONE_RUNNER_VERSION=${docker.ozone-runner.version} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml deleted file mode 100644 index 3f010a2d79a..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-compose.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - - 9882 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] - scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" - command: ["/opt/hadoop/bin/ozone","scm"] - recon: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9888:9888 - env_file: - - ./docker-config - environment: - WAITFOR: om:9874 - command: ["/opt/hadoop/bin/ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config deleted file mode 100644 index bf7871e0b5a..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon -OZONE-SITE.XML_ozone.recon.om.db.dir=/data/metadata/recon -OZONE-SITE.XML_ozone.handler.type=distributed -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.replication=3 -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -ASYNC_PROFILER_HOME=/opt/profiler - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone/README.md b/hadoop-ozone/dist/src/main/compose/ozone/README.md new file mode 100644 index 00000000000..c28f8324705 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/README.md @@ -0,0 +1,99 @@ + + +# Compose file with optional monitoring and profiling configs + +This directory contains a docker-compose definition for an Ozone cluster with all components (including S3 Gateway and Recon). + +There are two optional add-ons: + + * monitoring: adds Grafana, Jaeger and Prometheus sercvies, and configures Ozone to work with them + * profiling: allows sampling Ozone CPU/memory using [async-profiler](https://github.com/jvm-profiling-tools/async-profiler) + +## How to start + +TL;DR: + +1. single datanode: + ``` + ./run.sh -d + ``` +2. three datanodes for replication: + ``` + export OZONE_REPLICATION_FACTOR=3 + ./run.sh -d + ``` + +### Basics + +The cluster can be started with regular `docker-compose up` command. Use `-d` to start the cluster in the background. + +You can change the number of datanodes to start using the `--scale` option. Eg. to start 3 datanodes: `docker-compose up -d --scale datanode=3`. + +The cluster's replication factor (1 or 3) can be controlled by setting the `OZONE_REPLICATION_FACTOR` environment variable. It defaults to 1 to match the number of datanodes started by default, without the `--scale` option. + +For convenience the `run.sh` script can be used to make sure the replication factor and the number of datanodes match. It also passes any additional arguments provided on the command-line (eg. `-d`) to `docker-compose`. + +### Add-ons + +Monitoring and/or performance add-ons can be enabled via docker-compose's ability to use multiple compose files (by using the [`-f` option repeatedly](https://docs.docker.com/compose/reference/overview/#specifying-multiple-compose-files), or more easily by defining the [`COMPOSE_FILE` environment variable](https://docs.docker.com/compose/reference/envvars/#compose_file)): + +``` +# no COMPOSE_FILE var # => only Ozone +export COMPOSE_FILE=docker-compose.yaml:monitoring.yaml # => add monitoring +export COMPOSE_FILE=docker-compose.yaml:profiling.yaml # => add profiling +export COMPOSE_FILE=docker-compose.yaml:monitoring.yaml:profiling.yaml # => add both +``` + +Once the variable is defined, Ozone cluster with add-ons can be started/scaled/stopped etc. using the same `docker-compose` commands as for the base cluster. + +### Load generator + +Ozone comes with a load generator called Freon. + +You can enter one of the containers (eg. SCM) and start a Freon test: + +``` +docker-compose exec scm bash +ozone freon ockg -n1000 +``` + +You can also start two flavors of Freon as separate services, which allows scaling them up. Once all the datanodes are started, start Freon by adding its definition to `COMPOSE_FILE` and re-running the `docker-compose up` or `run.sh` command: + +``` +export COMPOSE_FILE="${COMPOSE_FILE}:freon-ockg.yaml" + +docker-compose up -d --no-recreate --scale datanode=3 +# OR +./run.sh -d +``` + +## How to use + +You can check the ozone web ui: + +OzoneManager: http://localhost:9874 +SCM: http://localhost:9876 + +### Monitoring + + * Prometheus: follows a pull based approach where metrics are published on an HTTP endpoint. Metrics can be checked on [Prometheus' web UI](http://localhost:9090/) + * Grafana: comes with two [dashboards](http://localhost:3000) for Ozone + * Ozone - Object Metrics + * Ozone - RPC Metrics + * Jaeger: collects distributed tracing information from Ozone, can be queried on the [Jaeger web UI](http://localhost:16686) + +### Profiling + +Start by hitting the `/prof` endpoint on the service to be profiled, eg. http://localhost:9876/prof for SCM. [Detailed instructions](https://cwiki.apache.org/confluence/display/HADOOP/Java+Profiling+of+Ozone) can be found in the Hadoop wiki. diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml index 6dc9d14ac5b..b38b7cd2aae 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -14,41 +14,59 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" + +# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) +x-common-config: + &common-config + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + +x-replication: + &replication + OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1} + services: - datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - - 9882 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] - scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true #required by the profiler - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" - command: ["/opt/hadoop/bin/ozone","scm"] + datanode: + <<: *common-config + ports: + - 9864 + - 9882 + environment: + <<: *replication + command: ["ozone","datanode"] + om: + <<: *common-config + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication + ports: + - 9874:9874 + command: ["ozone","om"] + scm: + <<: *common-config + ports: + - 9876:9876 + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + <<: *replication + command: ["ozone","scm"] + s3g: + <<: *common-config + environment: + <<: *replication + ports: + - 9878:9878 + command: ["ozone","s3g"] + recon: + <<: *common-config + ports: + - 9888:9888 + environment: + WAITFOR: om:9874 + <<: *replication + command: ["ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index d82cde2ea38..c4d79c14dfa 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -20,14 +20,7 @@ OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.handler.type=distributed +OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon +OZONE-SITE.XML_ozone.recon.om.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -ASYNC_PROFILER_HOME=/opt/profiler - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/freon-ockg.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml similarity index 71% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/freon-ockg.yaml rename to hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml index 72685bdc672..765827d44b7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/freon-ockg.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml @@ -14,12 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" services: - freon: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - ./docker-config - command: ["ozone","freon","ockg","-n100000"] + freon: + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + - monitoring.conf + environment: + - "OZONE-SITE.XML_ozone.replication=${OZONE_REPLICATION_FACTOR:-1}" + command: ["ozone","freon","ockg","-n100000"] diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/freon-rk.yaml b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml similarity index 72% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/freon-rk.yaml rename to hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml index 671bd229a32..7bb1dbf4b1f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/freon-rk.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml @@ -14,12 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" services: - freon: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - ./docker-config - command: ["ozone","freon","rk"] + freon: + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + - monitoring.conf + environment: + - "OZONE-SITE.XML_ozone.replication=${OZONE_REPLICATION_FACTOR:-1}" + command: ["ozone","freon","rk"] diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf similarity index 76% rename from hadoop-ozone/dist/src/main/compose/ozones3/docker-config rename to hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf index 75d197759b2..156d539d617 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf @@ -14,17 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.replication=3 -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds - +OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true +OZONE-SITE.XML_hdds.tracing.enabled=true HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 - +JAEGER_SAMPLER_PARAM=1 +JAEGER_SAMPLER_TYPE=const +JAEGER_AGENT_HOST=jaeger #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml new file mode 100644 index 00000000000..e08cc8ec5a0 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.4" + +x-monitoring-config: + &monitoring-config + env_file: + - monitoring.conf + +services: + jaeger: + image: jaegertracing/all-in-one:latest + environment: + COLLECTOR_ZIPKIN_HTTP_PORT: 9411 + ports: + - 16686:16686 + prometheus: + image: prom/prometheus + volumes: + - "./prometheus.yml:/etc/prometheus.yml" + command: ["--config.file","/etc/prometheus.yml"] + ports: + - 9090:9090 + grafana: + image: grafana/grafana + volumes: + - "../common/grafana/dashboards:/var/lib/grafana/dashboards" + - "../common/grafana/provisioning:/etc/grafana/provisioning" + - "../common/grafana/conf/grafana.ini:/etc/grafana/grafana.ini" + command: ["-config","/etc/grafana/grafana.ini"] + ports: + - 3000:3000 + datanode: + <<: *monitoring-config + om: + <<: *monitoring-config + scm: + <<: *monitoring-config + s3g: + <<: *monitoring-config + recon: + <<: *monitoring-config diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml old mode 100755 new mode 100644 similarity index 68% rename from hadoop-ozone/dist/src/main/compose/ozone-recon/test.sh rename to hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml index f4bfcc3d57f..f3f14f6288c --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,16 +14,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export COMPOSE_DIR +version: "3.4" -# shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" +x-profiling-config: + &profiling-config + privileged: true + environment: + - OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true + - ASYNC_PROFILER_HOME=/opt/profiler -start_docker_env - -execute_robot_test scm basic/basic.robot - -stop_docker_env - -generate_report +services: + datanode: + <<: *profiling-config + om: + <<: *profiling-config + scm: + <<: *profiling-config + s3g: + <<: *profiling-config + recon: + <<: *profiling-config diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml similarity index 72% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml rename to hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml index d4f475b0ff1..48e6ac5c660 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml +++ b/hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml @@ -23,13 +23,13 @@ scrape_configs: - targets: - "scm:9876" - "om:9874" - - "ozoneperf_datanode_1:9882" - - "ozoneperf_datanode_2:9882" - - "ozoneperf_datanode_3:9882" - - "ozoneperf_datanode_4:9882" - - "ozoneperf_datanode_5:9882" - - "ozoneperf_datanode_6:9882" - - "ozoneperf_datanode_7:9882" - - "ozoneperf_datanode_8:9882" - - "ozoneperf_datanode_9:9882" - - "ozoneperf_datanode_10:9882" + - "ozone_datanode_1:9882" + - "ozone_datanode_2:9882" + - "ozone_datanode_3:9882" + - "ozone_datanode_4:9882" + - "ozone_datanode_5:9882" + - "ozone_datanode_6:9882" + - "ozone_datanode_7:9882" + - "ozone_datanode_8:9882" + - "ozone_datanode_9:9882" + - "ozone_datanode_10:9882" diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/run.sh similarity index 75% rename from hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh rename to hadoop-ozone/dist/src/main/compose/ozone/run.sh index f4bfcc3d57f..f18aa78ac43 100755 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/run.sh @@ -15,16 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export COMPOSE_DIR +declare -ix OZONE_REPLICATION_FACTOR OZONE_SAFEMODE_MIN_DATANODES -# shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" +: ${OZONE_REPLICATION_FACTOR:=1} +: ${OZONE_SAFEMODE_MIN_DATANODES:=${OZONE_REPLICATION_FACTOR}} -start_docker_env - -execute_robot_test scm basic/basic.robot - -stop_docker_env - -generate_report +docker-compose up --scale datanode=${OZONE_REPLICATION_FACTOR} --no-recreate "$@" diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index e06f817f3d8..7decbd11e62 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -18,6 +18,9 @@ COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR +export SECURITY_ENABLED=false +export OZONE_REPLICATION_FACTOR=3 + # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" @@ -29,9 +32,11 @@ start_docker_env #Disabling for now, audit parser tool during parse getting exception. #execute_robot_test om auditparser -execute_robot_test scm basic/basic.robot +execute_robot_test scm basic + +execute_robot_test scm gdpr -execute_robot_test scm gdpr/gdpr.robot +execute_robot_test scm s3 stop_docker_env diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/.env b/hadoop-ozone/dist/src/main/compose/ozoneperf/.env deleted file mode 100644 index 96ab163b474..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/.env +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} -OZONE_RUNNER_VERSION=${docker.ozone-runner.version} diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md b/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md deleted file mode 100644 index d8aa8c3dcae..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md +++ /dev/null @@ -1,74 +0,0 @@ - - -# Compose files for local performance tests - -This directory contains docker-compose definition for an ozone cluster where -all the metrics are saved to a prometheus instance, and profiling and Jaeger -tracing is turned on and set up. - -Prometheus follows a pull based approach where the metrics are published - on a HTTP endpoint. - -Prometheus compatible metrics endpoint can be enabled by setting `hdds.prometheus.endpoint.enabled` property to `true` - -## How to start - -Start the cluster with `docker-compose` - -``` -docker-compose up -d -``` - -Scale datanodes up: - -``` -docker-compose up -d --scale datanode=3 -``` - -You can enter to the SCM container and start a freon test: - -``` -docker-compose exec scm bash -ozone freon ockg -n1000 -``` - -Or you can start freon instances in containers (to make it possible to scale them up): -If all the datanodes are started, start the freon instance: - -``` -docker-compose -f docker-compose.yaml -f freon-ockg.yaml up --scale datanode=3 -d -``` - -## How to use - -You can check the ozone web ui: - -OzoneManager: http://localhost:9874 -SCM: http://localhost:9876 - -You can check the ozone metrics from the prometheus web ui. - -http://localhost:9090/graph - -You can view Grafana dashboards at: - -http://localhost:3000 - -Default dashboards available are: -Ozone - Object Metrics -Ozone - RPC Metrics - -You can access the Jaeger UI at: -http://localhost:16686 \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml deleted file mode 100644 index b78cf098df3..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - command: ["ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["ozone","om"] - scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" - command: ["ozone","scm"] - jaeger: - image: jaegertracing/all-in-one:latest - environment: - COLLECTOR_ZIPKIN_HTTP_PORT: 9411 - ports: - - 16686:16686 - prometheus: - image: prom/prometheus - volumes: - - "./prometheus.yml:/etc/prometheus.yml" - command: ["--config.file","/etc/prometheus.yml"] - ports: - - 9090:9090 - grafana: - image: grafana/grafana - volumes: - - "../common/grafana/dashboards:/var/lib/grafana/dashboards" - - "../common/grafana/provisioning:/etc/grafana/provisioning" - - "../common/grafana/conf/grafana.ini:/etc/grafana/grafana.ini" - command: ["-config","/etc/grafana/grafana.ini"] - ports: - - 3000:3000 - s3g: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - privileged: true - volumes: - - ../..:/opt/hadoop - ports: - - 9878:9878 - env_file: - - ./docker-config - command: ["ozone","s3g"] diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config deleted file mode 100644 index 931f112508d..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_ozone.replication=3 -OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true -OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true - -ASYNC_PROFILER_HOME=/opt/profiler - -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 - -JAEGER_SAMPLER_PARAM=1 -JAEGER_SAMPLER_TYPE=const -JAEGER_AGENT_HOST=jaeger diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/.env b/hadoop-ozone/dist/src/main/compose/ozones3/.env deleted file mode 100644 index 96ab163b474..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozones3/.env +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} -OZONE_RUNNER_VERSION=${docker.ozone-runner.version} diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml deleted file mode 100644 index 26bcee5820a..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - command: ["ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["ozone","om"] - scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" - command: ["ozone","scm"] - s3g: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9878:9878 - env_file: - - ./docker-config - command: ["ozone","s3g"] diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh b/hadoop-ozone/dist/src/main/compose/ozones3/test.sh deleted file mode 100755 index a534a627f20..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export COMPOSE_DIR - -export SECURITY_ENABLED=false - -# shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" - -start_docker_env - -execute_robot_test scm basic/basic.robot - -execute_robot_test scm s3 - -stop_docker_env - -generate_report