diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java index 19a5a9cad5da..74c86706ede7 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java @@ -29,6 +29,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableQuantiles; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.util.MetricUtil; import java.util.Map; import java.util.UUID; @@ -82,6 +83,7 @@ public static synchronized void release() { } referenceCount--; if (referenceCount == 0) { + instance.stop(); DefaultMetricsSystem.instance().unregisterSource( SOURCE_NAME + instanceCount); instance = null; @@ -140,6 +142,17 @@ private ContainerClientMetrics() { } } + public void stop() { + MetricUtil.stop(listBlockLatency); + MetricUtil.stop(getBlockLatency); + MetricUtil.stop(getCommittedBlockLengthLatency); + MetricUtil.stop(readChunkLatency); + MetricUtil.stop(getSmallFileLatency); + MetricUtil.stop(hsyncLatencyNs); + MetricUtil.stop(omHsyncLatencyNs); + MetricUtil.stop(datanodeHsyncLatencyNs); + } + public void recordWriteChunk(Pipeline pipeline, long chunkSizeBytes) { writeChunkCallsByPipeline.computeIfAbsent(pipeline.getId(), pipelineID -> registry.newCounter( diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java index 704e886659ab..50e010e85a2f 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; @@ -129,6 +130,7 @@ public void reset() { } public void unRegister() { + IOUtils.closeQuietly(containerOpsLatency.values()); MetricsSystem ms = DefaultMetricsSystem.instance(); ms.unregisterSource(SOURCE_NAME); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java index 6bd83b44a93f..6e0dde669867 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java @@ -33,6 +33,7 @@ import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.util.MetricUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,6 +101,8 @@ public static synchronized GrpcMetrics create(Configuration conf) { */ public void unRegister() { DefaultMetricsSystem.instance().unregisterSource(SOURCE_NAME); + MetricUtil.stop(grpcProcessingTimeMillisQuantiles); + MetricUtil.stop(grpcQueueTimeMillisQuantiles); } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java index 23ff3c0f29ed..9d903b900acf 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java @@ -110,4 +110,20 @@ public static List createQuantiles(MetricsRegistry registry, sampleName, valueName, interval); }).collect(Collectors.toList()); } + + public static void stop(MutableQuantiles... quantiles) { + if (quantiles != null) { + stop(Arrays.asList(quantiles)); + } + } + + public static void stop(Iterable quantiles) { + if (quantiles != null) { + for (MutableQuantiles q : quantiles) { + if (q != null) { + q.stop(); + } + } + } + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java index 3f5150bd62cd..39e887eaa499 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java @@ -22,7 +22,9 @@ import org.apache.hadoop.metrics2.lib.MutableQuantiles; import org.apache.hadoop.metrics2.lib.MutableStat; +import java.io.Closeable; import java.util.List; +import java.util.Map; /** * The {@code PerformanceMetrics} class encapsulates a collection of related @@ -30,7 +32,7 @@ * This class provides methods to update these metrics and to * snapshot their values for reporting. */ -public class PerformanceMetrics { +public class PerformanceMetrics implements Closeable { private final MutableStat stat; private final List quantiles; private final MutableMinMax minMax; @@ -43,12 +45,13 @@ public class PerformanceMetrics { * @param intervals the intervals for quantiles computation. Note, each * interval in 'intervals' increases memory usage, as it corresponds * to a separate quantile calculator. + * @return {@link PerformanceMetrics} instances created, mapped by field name */ - public static synchronized void initializeMetrics(T source, + public static synchronized Map initializeMetrics(T source, MetricsRegistry registry, String sampleName, String valueName, int[] intervals) { try { - PerformanceMetricsInitializer.initialize( + return PerformanceMetricsInitializer.initialize( source, registry, sampleName, valueName, intervals); } catch (IllegalAccessException e) { throw new RuntimeException("Failed to initialize PerformanceMetrics", e); @@ -73,6 +76,11 @@ public PerformanceMetrics( minMax = new MutableMinMax(registry, name, description, valueName); } + @Override + public void close() { + MetricUtil.stop(quantiles); + } + /** * Adds a value to all the aggregated metrics. * @@ -95,6 +103,5 @@ public void snapshot(MetricsRecordBuilder recordBuilder, boolean all) { this.quantiles.forEach(quantile -> quantile.snapshot(recordBuilder, all)); this.minMax.snapshot(recordBuilder, all); } - } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java index b2e83bb780cf..cb6f77e9f5c6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java @@ -21,6 +21,8 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; /** * Utility class for initializing PerformanceMetrics in a MetricsSource. @@ -36,11 +38,13 @@ private PerformanceMetricsInitializer() { } * @param sampleName sample name * @param valueName value name * @param intervals intervals for quantiles + * @return {@link PerformanceMetrics} instances created, mapped by field name * @throws IllegalAccessException if unable to access the field */ - public static void initialize(T source, MetricsRegistry registry, + public static Map initialize(T source, MetricsRegistry registry, String sampleName, String valueName, int[] intervals) throws IllegalAccessException { + Map instances = new HashMap<>(); Field[] fields = source.getClass().getDeclaredFields(); for (Field field : fields) { @@ -54,8 +58,11 @@ public static void initialize(T source, MetricsRegistry registry, sampleName, valueName, intervals); field.setAccessible(true); field.set(source, performanceMetrics); + instances.put(name, performanceMetrics); } } } + + return instances; } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java index 883f6cd851e3..46887441f0aa 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java @@ -30,7 +30,9 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableQuantiles; import org.apache.hadoop.metrics2.lib.MutableRate; +import org.apache.hadoop.ozone.util.MetricUtil; +import java.io.Closeable; import java.util.EnumMap; /** @@ -47,7 +49,7 @@ */ @InterfaceAudience.Private @Metrics(about = "Storage Container DataNode Metrics", context = "dfs") -public class ContainerMetrics { +public class ContainerMetrics implements Closeable { public static final String STORAGE_CONTAINER_METRICS = "StorageContainerMetrics"; @Metric private MutableCounterLong numOps; @@ -105,6 +107,11 @@ public static void remove() { ms.unregisterSource(STORAGE_CONTAINER_METRICS); } + @Override + public void close() { + opsLatQuantiles.values().forEach(MetricUtil::stop); + } + public void incContainerOpsMetrics(ContainerProtos.Type type) { numOps.incr(); numOpsArray.get(type).incr(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index e40fa635c121..62196cdd87f2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.security.token.TokenVerifier; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService; @@ -512,6 +513,7 @@ public void stop() { } blockDeletingService.shutdown(); recoveringContainerScrubbingService.shutdown(); + IOUtils.closeQuietly(metrics); ContainerMetrics.remove(); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java index dd84d019176e..ea34be638eb7 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; @@ -33,12 +34,15 @@ import org.apache.hadoop.ozone.util.PerformanceMetrics; import org.apache.hadoop.util.Time; +import java.io.Closeable; +import java.util.Map; + /** * This class maintains S3 Gateway related metrics. */ @InterfaceAudience.Private @Metrics(about = "S3 Gateway Metrics", context = OzoneConsts.OZONE) -public final class S3GatewayMetrics implements MetricsSource { +public final class S3GatewayMetrics implements Closeable, MetricsSource { public static final String SOURCE_NAME = S3GatewayMetrics.class.getSimpleName(); @@ -242,6 +246,8 @@ public final class S3GatewayMetrics implements MetricsSource { @Metric(about = "Latency for copy metadata of an key in nanoseconds") private PerformanceMetrics copyKeyMetadataLatencyNs; + private final Map performanceMetrics; + /** * Private constructor. */ @@ -249,10 +255,15 @@ private S3GatewayMetrics(OzoneConfiguration conf) { this.registry = new MetricsRegistry(SOURCE_NAME); int[] intervals = conf.getInts(S3GatewayConfigKeys .OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY); - PerformanceMetrics.initializeMetrics( + performanceMetrics = PerformanceMetrics.initializeMetrics( this, registry, "Ops", "Time", intervals); } + @Override + public void close() { + IOUtils.closeQuietly(performanceMetrics.values()); + } + /** * Create and returns S3 Gateway Metrics instance. * @@ -272,6 +283,7 @@ public static synchronized S3GatewayMetrics create(OzoneConfiguration conf) { * Unregister the metrics instance. */ public static void unRegister() { + IOUtils.closeQuietly(instance); instance = null; MetricsSystem ms = DefaultMetricsSystem.instance(); ms.unregisterSource(SOURCE_NAME); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java index d0c9a33b330f..877e25ac7ccd 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java @@ -75,7 +75,8 @@ DatanodeSimulator.class, OmMetadataGenerator.class, DNRPCLoadGenerator.class, - HsyncGenerator.class + HsyncGenerator.class, + OzoneClientCreator.class, }, versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientCreator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientCreator.java new file mode 100644 index 000000000000..2fc4cb48eac4 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientCreator.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.freon; + +import com.codahale.metrics.Timer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import picocli.CommandLine; + +import java.util.concurrent.Callable; + +/** + * Creates and closes Ozone clients. + */ +@CommandLine.Command(name = "occ", + aliases = "ozone-client-creator", + description = "Create and close Ozone clients without doing anything useful", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + showDefaultValues = true) +public class OzoneClientCreator extends BaseFreonGenerator implements Callable { + + @CommandLine.Option(names = "--om-service-id", + description = "OM Service ID" + ) + private String omServiceID; + + private Timer timer; + private OzoneConfiguration conf; + + @Override + public Void call() { + init(); + conf = createOzoneConfiguration(); + timer = getMetrics().timer("client-create"); + runTests(this::createClient); + return null; + } + + private void createClient(long step) { + timer.time(this::createClientSafely); + } + + private void createClientSafely() { + try { + createOzoneClient(omServiceID, conf).close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +}