diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeCleanUpGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeCleanUpGlueMetastore.java index 7a1ffd7ab399..d6716dd463d3 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeCleanUpGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeCleanUpGlueMetastore.java @@ -20,7 +20,7 @@ import com.amazonaws.services.glue.model.GetDatabasesRequest; import com.amazonaws.services.glue.model.GetDatabasesResult; import io.airlift.log.Logger; -import io.trino.plugin.hive.metastore.glue.GlueMetastoreApiStats; +import io.trino.plugin.hive.aws.AwsApiCallStats; import org.testng.annotations.Test; import java.util.List; @@ -46,7 +46,7 @@ public void cleanupOrphanedDatabases() new GetDatabasesRequest(), GetDatabasesRequest::setNextToken, GetDatabasesResult::getNextToken, - new GlueMetastoreApiStats()) + new AwsApiCallStats()) .map(GetDatabasesResult::getDatabaseList) .flatMap(List::stream) .filter(glueDatabase -> glueDatabase.getName().startsWith(TEST_DATABASE_NAME_PREFIX) && diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AbstractSdkMetricsCollector.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AbstractSdkMetricsCollector.java deleted file mode 100644 index 070abc3feb8b..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AbstractSdkMetricsCollector.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.aws; - -import com.amazonaws.Request; -import com.amazonaws.Response; -import com.amazonaws.metrics.RequestMetricCollector; -import com.amazonaws.util.AWSRequestMetrics; -import com.amazonaws.util.TimingInfo; -import io.airlift.units.Duration; - -import java.util.List; -import java.util.function.Consumer; - -import static com.amazonaws.util.AWSRequestMetrics.Field.ClientExecuteTime; -import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolAvailableCount; -import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolLeasedCount; -import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolPendingCount; -import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientRetryCount; -import static com.amazonaws.util.AWSRequestMetrics.Field.HttpRequestTime; -import static com.amazonaws.util.AWSRequestMetrics.Field.RequestCount; -import static com.amazonaws.util.AWSRequestMetrics.Field.RetryPauseTime; -import static com.amazonaws.util.AWSRequestMetrics.Field.ThrottleException; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public abstract class AbstractSdkMetricsCollector - extends RequestMetricCollector -{ - @Override - public final void collectMetrics(Request request, Response response) - { - TimingInfo timingInfo = request.getAWSRequestMetrics().getTimingInfo(); - - Number requestCounts = timingInfo.getCounter(RequestCount.name()); - if (requestCounts != null) { - recordRequestCount(requestCounts.longValue()); - } - - Number retryCounts = timingInfo.getCounter(HttpClientRetryCount.name()); - if (retryCounts != null) { - recordRetryCount(retryCounts.longValue()); - } - - Number throttleExceptions = timingInfo.getCounter(ThrottleException.name()); - if (throttleExceptions != null) { - recordThrottleExceptionCount(throttleExceptions.longValue()); - } - - Number httpClientPoolAvailableCount = timingInfo.getCounter(HttpClientPoolAvailableCount.name()); - if (httpClientPoolAvailableCount != null) { - recordHttpClientPoolAvailableCount(httpClientPoolAvailableCount.longValue()); - } - - Number httpClientPoolLeasedCount = timingInfo.getCounter(HttpClientPoolLeasedCount.name()); - if (httpClientPoolLeasedCount != null) { - recordHttpClientPoolLeasedCount(httpClientPoolLeasedCount.longValue()); - } - - Number httpClientPoolPendingCount = timingInfo.getCounter(HttpClientPoolPendingCount.name()); - if (httpClientPoolPendingCount != null) { - recordHttpClientPoolPendingCount(httpClientPoolPendingCount.longValue()); - } - - recordSubTimingDurations(timingInfo, HttpRequestTime, this::recordHttpRequestTime); - recordSubTimingDurations(timingInfo, ClientExecuteTime, this::recordClientExecutionTime); - recordSubTimingDurations(timingInfo, RetryPauseTime, this::recordRetryPauseTime); - } - - protected abstract void recordRequestCount(long count); - - protected abstract void recordRetryCount(long count); - - protected abstract void recordThrottleExceptionCount(long count); - - protected abstract void recordHttpRequestTime(Duration duration); - - protected abstract void recordClientExecutionTime(Duration duration); - - protected abstract void recordRetryPauseTime(Duration duration); - - protected abstract void recordHttpClientPoolAvailableCount(long count); - - protected abstract void recordHttpClientPoolLeasedCount(long count); - - protected abstract void recordHttpClientPoolPendingCount(long count); - - private static void recordSubTimingDurations(TimingInfo timingInfo, AWSRequestMetrics.Field field, Consumer consumer) - { - List subTimings = timingInfo.getAllSubMeasurements(field.name()); - if (subTimings != null) { - for (TimingInfo subTiming : subTimings) { - Double millis = subTiming.getTimeTakenMillisIfKnown(); - if (millis != null) { - consumer.accept(new Duration(millis, MILLISECONDS)); - } - } - } - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreApiStats.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AwsApiCallStats.java similarity index 89% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreApiStats.java rename to plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AwsApiCallStats.java index e774832e1361..a1b2d1456c85 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreApiStats.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AwsApiCallStats.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive.metastore.glue; +package io.trino.plugin.hive.aws; import io.airlift.stats.CounterStat; import io.airlift.stats.TimeStat; @@ -25,7 +25,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; @ThreadSafe -public class GlueMetastoreApiStats +public class AwsApiCallStats { private final TimeStat time = new TimeStat(MILLISECONDS); private final CounterStat totalFailures = new CounterStat(); @@ -56,9 +56,9 @@ public CounterStat getTotalFailures() return totalFailures; } - public void recordCall(long executionTimeInMillis, boolean failure) + public void recordCall(long executionTimeNanos, boolean failure) { - time.add(executionTimeInMillis, MILLISECONDS); + time.addNanos(executionTimeNanos); if (failure) { totalFailures.update(1); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AwsSdkClientCoreStats.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AwsSdkClientCoreStats.java new file mode 100644 index 000000000000..f77df3a512fe --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/aws/AwsSdkClientCoreStats.java @@ -0,0 +1,184 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.aws; + +import com.amazonaws.Request; +import com.amazonaws.Response; +import com.amazonaws.metrics.RequestMetricCollector; +import com.amazonaws.util.AWSRequestMetrics; +import com.amazonaws.util.TimingInfo; +import io.airlift.stats.CounterStat; +import io.airlift.stats.TimeStat; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import static com.amazonaws.util.AWSRequestMetrics.Field.ClientExecuteTime; +import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolAvailableCount; +import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolLeasedCount; +import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolPendingCount; +import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientRetryCount; +import static com.amazonaws.util.AWSRequestMetrics.Field.HttpRequestTime; +import static com.amazonaws.util.AWSRequestMetrics.Field.RequestCount; +import static com.amazonaws.util.AWSRequestMetrics.Field.RetryPauseTime; +import static com.amazonaws.util.AWSRequestMetrics.Field.ThrottleException; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +@ThreadSafe +public final class AwsSdkClientCoreStats +{ + private final CounterStat awsRequestCount = new CounterStat(); + private final CounterStat awsRetryCount = new CounterStat(); + private final CounterStat awsThrottleExceptions = new CounterStat(); + private final TimeStat awsRequestTime = new TimeStat(MILLISECONDS); + private final TimeStat awsClientExecuteTime = new TimeStat(MILLISECONDS); + private final TimeStat awsClientRetryPauseTime = new TimeStat(MILLISECONDS); + private final AtomicLong awsHttpClientPoolAvailableCount = new AtomicLong(); + private final AtomicLong awsHttpClientPoolLeasedCount = new AtomicLong(); + private final AtomicLong awsHttpClientPoolPendingCount = new AtomicLong(); + + @Managed + @Nested + public CounterStat getAwsRequestCount() + { + return awsRequestCount; + } + + @Managed + @Nested + public CounterStat getAwsRetryCount() + { + return awsRetryCount; + } + + @Managed + @Nested + public CounterStat getAwsThrottleExceptions() + { + return awsThrottleExceptions; + } + + @Managed + @Nested + public TimeStat getAwsRequestTime() + { + return awsRequestTime; + } + + @Managed + @Nested + public TimeStat getAwsClientExecuteTime() + { + return awsClientExecuteTime; + } + + @Managed + @Nested + public TimeStat getAwsClientRetryPauseTime() + { + return awsClientRetryPauseTime; + } + + @Managed + public long getAwsHttpClientPoolAvailableCount() + { + return awsHttpClientPoolAvailableCount.get(); + } + + @Managed + public long getAwsHttpClientPoolLeasedCount() + { + return awsHttpClientPoolLeasedCount.get(); + } + + @Managed + public long getAwsHttpClientPoolPendingCount() + { + return awsHttpClientPoolPendingCount.get(); + } + + public AwsSdkClientCoreRequestMetricCollector newRequestMetricCollector() + { + return new AwsSdkClientCoreRequestMetricCollector(this); + } + + public static class AwsSdkClientCoreRequestMetricCollector + extends RequestMetricCollector + { + private final AwsSdkClientCoreStats stats; + + protected AwsSdkClientCoreRequestMetricCollector(AwsSdkClientCoreStats stats) + { + this.stats = requireNonNull(stats, "stats is null"); + } + + @Override + public void collectMetrics(Request request, Response response) + { + TimingInfo timingInfo = request.getAWSRequestMetrics().getTimingInfo(); + + Number requestCounts = timingInfo.getCounter(RequestCount.name()); + if (requestCounts != null) { + stats.awsRequestCount.update(requestCounts.longValue()); + } + + Number retryCounts = timingInfo.getCounter(HttpClientRetryCount.name()); + if (retryCounts != null) { + stats.awsRetryCount.update(retryCounts.longValue()); + } + + Number throttleExceptions = timingInfo.getCounter(ThrottleException.name()); + if (throttleExceptions != null) { + stats.awsThrottleExceptions.update(throttleExceptions.longValue()); + } + + Number httpClientPoolAvailableCount = timingInfo.getCounter(HttpClientPoolAvailableCount.name()); + if (httpClientPoolAvailableCount != null) { + stats.awsHttpClientPoolAvailableCount.set(httpClientPoolAvailableCount.longValue()); + } + + Number httpClientPoolLeasedCount = timingInfo.getCounter(HttpClientPoolLeasedCount.name()); + if (httpClientPoolLeasedCount != null) { + stats.awsHttpClientPoolLeasedCount.set(httpClientPoolLeasedCount.longValue()); + } + + Number httpClientPoolPendingCount = timingInfo.getCounter(HttpClientPoolPendingCount.name()); + if (httpClientPoolPendingCount != null) { + stats.awsHttpClientPoolPendingCount.set(httpClientPoolPendingCount.longValue()); + } + + recordSubTimingDurations(timingInfo, HttpRequestTime, stats.awsRequestTime); + recordSubTimingDurations(timingInfo, ClientExecuteTime, stats.awsClientExecuteTime); + recordSubTimingDurations(timingInfo, RetryPauseTime, stats.awsClientRetryPauseTime); + } + + private static void recordSubTimingDurations(TimingInfo timingInfo, AWSRequestMetrics.Field field, TimeStat timeStat) + { + List subTimings = timingInfo.getAllSubMeasurements(field.name()); + if (subTimings != null) { + for (TimingInfo subTiming : subTimings) { + Long endTimeNanos = subTiming.getEndTimeNanoIfKnown(); + if (endTimeNanos != null) { + timeStat.addNanos(endTimeNanos - subTiming.getStartTimeNano()); + } + } + } + } + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/AwsSdkUtil.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/AwsSdkUtil.java index 02a6962fdee1..646252513c11 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/AwsSdkUtil.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/AwsSdkUtil.java @@ -14,6 +14,7 @@ package io.trino.plugin.hive.metastore.glue; import com.google.common.collect.AbstractIterator; +import io.trino.plugin.hive.aws.AwsApiCallStats; import java.util.Iterator; import java.util.function.BiConsumer; @@ -38,7 +39,7 @@ public static Stream getPaginatedResults( Request request, BiConsumer setNextToken, Function extractNextToken, - GlueMetastoreApiStats stats) + AwsApiCallStats stats) { requireNonNull(submission, "submission is null"); requireNonNull(request, "request is null"); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java index 7f6d42f82966..5a1a8a01321a 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java @@ -76,6 +76,7 @@ import io.trino.plugin.hive.SchemaAlreadyExistsException; import io.trino.plugin.hive.TableAlreadyExistsException; import io.trino.plugin.hive.acid.AcidTransaction; +import io.trino.plugin.hive.aws.AwsApiCallStats; import io.trino.plugin.hive.metastore.Column; import io.trino.plugin.hive.metastore.Database; import io.trino.plugin.hive.metastore.HiveColumnStatistics; @@ -146,7 +147,7 @@ import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.spi.security.PrincipalType.USER; import static java.util.Objects.requireNonNull; -import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.function.Predicate.not; import static java.util.function.UnaryOperator.identity; import static java.util.stream.Collectors.toCollection; @@ -1128,10 +1129,10 @@ public Set listTablePrivileges(String databaseName, String ta static class StatsRecordingAsyncHandler implements AsyncHandler { - private final GlueMetastoreApiStats stats; + private final AwsApiCallStats stats; private final Stopwatch stopwatch; - public StatsRecordingAsyncHandler(GlueMetastoreApiStats stats) + public StatsRecordingAsyncHandler(AwsApiCallStats stats) { this.stats = requireNonNull(stats, "stats is null"); this.stopwatch = Stopwatch.createStarted(); @@ -1140,13 +1141,13 @@ public StatsRecordingAsyncHandler(GlueMetastoreApiStats stats) @Override public void onError(Exception e) { - stats.recordCall(stopwatch.elapsed(MILLISECONDS), true); + stats.recordCall(stopwatch.elapsed(NANOSECONDS), true); } @Override public void onSuccess(AmazonWebServiceRequest request, Object o) { - stats.recordCall(stopwatch.elapsed(MILLISECONDS), false); + stats.recordCall(stopwatch.elapsed(NANOSECONDS), false); } } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreStats.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreStats.java index bbc525a19029..7067de2efaa5 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreStats.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueMetastoreStats.java @@ -13,359 +13,227 @@ */ package io.trino.plugin.hive.metastore.glue; -import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeStat; -import io.airlift.units.Duration; -import io.trino.plugin.hive.aws.AbstractSdkMetricsCollector; +import com.amazonaws.metrics.RequestMetricCollector; +import io.trino.plugin.hive.aws.AwsApiCallStats; +import io.trino.plugin.hive.aws.AwsSdkClientCoreStats; +import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; -import java.util.concurrent.atomic.AtomicLong; - -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - public class GlueMetastoreStats { - private final GlueMetastoreApiStats getDatabases = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getDatabase = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getTables = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getTable = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats createDatabase = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats deleteDatabase = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats updateDatabase = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats createTable = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats deleteTable = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats updateTable = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getPartitionNames = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getPartitions = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getPartition = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getPartitionByName = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats createPartitions = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats deletePartition = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats updatePartition = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats batchUpdatePartition = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats batchCreatePartition = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getColumnStatisticsForTable = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats getColumnStatisticsForPartition = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats updateColumnStatisticsForTable = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats deleteColumnStatisticsForTable = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats updateColumnStatisticsForPartition = new GlueMetastoreApiStats(); - private final GlueMetastoreApiStats deleteColumnStatisticsForPartition = new GlueMetastoreApiStats(); - - // see AWSRequestMetrics - private final CounterStat awsRequestCount = new CounterStat(); - private final CounterStat awsRetryCount = new CounterStat(); - private final CounterStat awsThrottleExceptions = new CounterStat(); - private final TimeStat awsRequestTime = new TimeStat(MILLISECONDS); - private final TimeStat awsClientExecuteTime = new TimeStat(MILLISECONDS); - private final TimeStat awsClientRetryPauseTime = new TimeStat(MILLISECONDS); - private final AtomicLong awsHttpClientPoolAvailableCount = new AtomicLong(); - private final AtomicLong awsHttpClientPoolLeasedCount = new AtomicLong(); - private final AtomicLong awsHttpClientPoolPendingCount = new AtomicLong(); - - @Managed - @Nested - public GlueMetastoreApiStats getGetDatabases() + private final AwsApiCallStats getDatabases = new AwsApiCallStats(); + private final AwsApiCallStats getDatabase = new AwsApiCallStats(); + private final AwsApiCallStats getTables = new AwsApiCallStats(); + private final AwsApiCallStats getTable = new AwsApiCallStats(); + private final AwsApiCallStats createDatabase = new AwsApiCallStats(); + private final AwsApiCallStats deleteDatabase = new AwsApiCallStats(); + private final AwsApiCallStats updateDatabase = new AwsApiCallStats(); + private final AwsApiCallStats createTable = new AwsApiCallStats(); + private final AwsApiCallStats deleteTable = new AwsApiCallStats(); + private final AwsApiCallStats updateTable = new AwsApiCallStats(); + private final AwsApiCallStats getPartitionNames = new AwsApiCallStats(); + private final AwsApiCallStats getPartitions = new AwsApiCallStats(); + private final AwsApiCallStats getPartition = new AwsApiCallStats(); + private final AwsApiCallStats getPartitionByName = new AwsApiCallStats(); + private final AwsApiCallStats createPartitions = new AwsApiCallStats(); + private final AwsApiCallStats deletePartition = new AwsApiCallStats(); + private final AwsApiCallStats updatePartition = new AwsApiCallStats(); + private final AwsApiCallStats batchUpdatePartition = new AwsApiCallStats(); + private final AwsApiCallStats batchCreatePartition = new AwsApiCallStats(); + private final AwsApiCallStats getColumnStatisticsForTable = new AwsApiCallStats(); + private final AwsApiCallStats getColumnStatisticsForPartition = new AwsApiCallStats(); + private final AwsApiCallStats updateColumnStatisticsForTable = new AwsApiCallStats(); + private final AwsApiCallStats deleteColumnStatisticsForTable = new AwsApiCallStats(); + private final AwsApiCallStats updateColumnStatisticsForPartition = new AwsApiCallStats(); + private final AwsApiCallStats deleteColumnStatisticsForPartition = new AwsApiCallStats(); + + private final AwsSdkClientCoreStats clientCoreStats = new AwsSdkClientCoreStats(); + + @Managed + @Nested + public AwsApiCallStats getGetDatabases() { return getDatabases; } @Managed @Nested - public GlueMetastoreApiStats getGetDatabase() + public AwsApiCallStats getGetDatabase() { return getDatabase; } @Managed @Nested - public GlueMetastoreApiStats getGetTables() + public AwsApiCallStats getGetTables() { return getTables; } @Managed @Nested - public GlueMetastoreApiStats getGetTable() + public AwsApiCallStats getGetTable() { return getTable; } @Managed @Nested - public GlueMetastoreApiStats getCreateDatabase() + public AwsApiCallStats getCreateDatabase() { return createDatabase; } @Managed @Nested - public GlueMetastoreApiStats getDeleteDatabase() + public AwsApiCallStats getDeleteDatabase() { return deleteDatabase; } @Managed @Nested - public GlueMetastoreApiStats getUpdateDatabase() + public AwsApiCallStats getUpdateDatabase() { return updateDatabase; } @Managed @Nested - public GlueMetastoreApiStats getCreateTable() + public AwsApiCallStats getCreateTable() { return createTable; } @Managed @Nested - public GlueMetastoreApiStats getDeleteTable() + public AwsApiCallStats getDeleteTable() { return deleteTable; } @Managed @Nested - public GlueMetastoreApiStats getUpdateTable() + public AwsApiCallStats getUpdateTable() { return updateTable; } @Managed @Nested - public GlueMetastoreApiStats getGetPartitionNames() + public AwsApiCallStats getGetPartitionNames() { return getPartitionNames; } @Managed @Nested - public GlueMetastoreApiStats getGetPartitions() + public AwsApiCallStats getGetPartitions() { return getPartitions; } @Managed @Nested - public GlueMetastoreApiStats getGetPartition() + public AwsApiCallStats getGetPartition() { return getPartition; } @Managed @Nested - public GlueMetastoreApiStats getGetPartitionByName() + public AwsApiCallStats getGetPartitionByName() { return getPartitionByName; } @Managed @Nested - public GlueMetastoreApiStats getCreatePartitions() + public AwsApiCallStats getCreatePartitions() { return createPartitions; } @Managed @Nested - public GlueMetastoreApiStats getDeletePartition() + public AwsApiCallStats getDeletePartition() { return deletePartition; } @Managed @Nested - public GlueMetastoreApiStats getUpdatePartition() + public AwsApiCallStats getUpdatePartition() { return updatePartition; } @Managed @Nested - public GlueMetastoreApiStats getBatchUpdatePartition() + public AwsApiCallStats getBatchUpdatePartition() { return batchUpdatePartition; } @Managed @Nested - public GlueMetastoreApiStats getBatchCreatePartition() + public AwsApiCallStats getBatchCreatePartition() { return batchCreatePartition; } @Managed @Nested - public GlueMetastoreApiStats getGetColumnStatisticsForTable() + public AwsApiCallStats getGetColumnStatisticsForTable() { return getColumnStatisticsForTable; } @Managed @Nested - public GlueMetastoreApiStats getGetColumnStatisticsForPartition() + public AwsApiCallStats getGetColumnStatisticsForPartition() { return getColumnStatisticsForPartition; } @Managed @Nested - public GlueMetastoreApiStats getUpdateColumnStatisticsForTable() + public AwsApiCallStats getUpdateColumnStatisticsForTable() { return updateColumnStatisticsForTable; } @Managed @Nested - public GlueMetastoreApiStats getDeleteColumnStatisticsForTable() + public AwsApiCallStats getDeleteColumnStatisticsForTable() { return deleteColumnStatisticsForTable; } @Managed @Nested - public GlueMetastoreApiStats getUpdateColumnStatisticsForPartition() + public AwsApiCallStats getUpdateColumnStatisticsForPartition() { return updateColumnStatisticsForPartition; } @Managed @Nested - public GlueMetastoreApiStats getDeleteColumnStatisticsForPartition() + public AwsApiCallStats getDeleteColumnStatisticsForPartition() { return deleteColumnStatisticsForPartition; } @Managed - @Nested - public CounterStat getAwsRequestCount() + @Flatten + public AwsSdkClientCoreStats getClientCoreStats() { - return awsRequestCount; + return clientCoreStats; } - @Managed - @Nested - public CounterStat getAwsRetryCount() - { - return awsRetryCount; - } - - @Managed - @Nested - public CounterStat getAwsThrottleExceptions() - { - return awsThrottleExceptions; - } - - @Managed - @Nested - public TimeStat getAwsRequestTime() - { - return awsRequestTime; - } - - @Managed - @Nested - public TimeStat getAwsClientExecuteTime() - { - return awsClientExecuteTime; - } - - @Managed - @Nested - public TimeStat getAwsClientRetryPauseTime() + public RequestMetricCollector newRequestMetricsCollector() { - return awsClientRetryPauseTime; - } - - @Managed - public long getAwsHttpClientPoolAvailableCount() - { - return awsHttpClientPoolAvailableCount.get(); - } - - @Managed - public long getAwsHttpClientPoolLeasedCount() - { - return awsHttpClientPoolLeasedCount.get(); - } - - @Managed - public long getAwsHttpClientPoolPendingCount() - { - return awsHttpClientPoolPendingCount.get(); - } - - public GlueSdkClientMetricsCollector newRequestMetricsCollector() - { - return new GlueSdkClientMetricsCollector(this); - } - - public static class GlueSdkClientMetricsCollector - extends AbstractSdkMetricsCollector - { - private final GlueMetastoreStats stats; - - public GlueSdkClientMetricsCollector(GlueMetastoreStats stats) - { - this.stats = requireNonNull(stats, "stats is null"); - } - - @Override - protected void recordRequestCount(long count) - { - stats.awsRequestCount.update(count); - } - - @Override - protected void recordRetryCount(long count) - { - stats.awsRetryCount.update(count); - } - - @Override - protected void recordThrottleExceptionCount(long count) - { - stats.awsThrottleExceptions.update(count); - } - - @Override - protected void recordHttpRequestTime(Duration duration) - { - stats.awsRequestTime.add(duration); - } - - @Override - protected void recordClientExecutionTime(Duration duration) - { - stats.awsClientExecuteTime.add(duration); - } - - @Override - protected void recordRetryPauseTime(Duration duration) - { - stats.awsClientRetryPauseTime.add(duration); - } - - @Override - protected void recordHttpClientPoolAvailableCount(long count) - { - stats.awsHttpClientPoolAvailableCount.set(count); - } - - @Override - protected void recordHttpClientPoolLeasedCount(long count) - { - stats.awsHttpClientPoolLeasedCount.set(count); - } - - @Override - protected void recordHttpClientPoolPendingCount(long count) - { - stats.awsHttpClientPoolPendingCount.set(count); - } + return clientCoreStats.newRequestMetricCollector(); } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystem.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystem.java index 2c7877d2c9d9..db2ff8a03a41 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystem.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystem.java @@ -219,7 +219,7 @@ public class TrinoS3FileSystem private static final Logger log = Logger.get(TrinoS3FileSystem.class); private static final TrinoS3FileSystemStats STATS = new TrinoS3FileSystemStats(); - private static final RequestMetricCollector METRIC_COLLECTOR = new TrinoS3FileSystemMetricCollector(STATS); + private static final RequestMetricCollector METRIC_COLLECTOR = STATS.newRequestMetricCollector(); private static final String DIRECTORY_SUFFIX = "_$folder$"; private static final DataSize BLOCK_SIZE = DataSize.of(32, MEGABYTE); private static final DataSize MAX_SKIP_SIZE = DataSize.of(1, MEGABYTE); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystemMetricCollector.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystemMetricCollector.java deleted file mode 100644 index cfd09b729433..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystemMetricCollector.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.s3; - -import io.airlift.units.Duration; -import io.trino.plugin.hive.aws.AbstractSdkMetricsCollector; - -import static java.util.Objects.requireNonNull; - -public class TrinoS3FileSystemMetricCollector - extends AbstractSdkMetricsCollector -{ - private final TrinoS3FileSystemStats stats; - - public TrinoS3FileSystemMetricCollector(TrinoS3FileSystemStats stats) - { - this.stats = requireNonNull(stats, "stats is null"); - } - - @Override - protected void recordRequestCount(long count) - { - stats.updateAwsRequestCount(count); - } - - @Override - protected void recordRetryCount(long count) - { - stats.updateAwsRetryCount(count); - } - - @Override - protected void recordThrottleExceptionCount(long count) - { - stats.updateAwsThrottleExceptionsCount(count); - } - - @Override - protected void recordHttpRequestTime(Duration duration) - { - stats.addAwsRequestTime(duration); - } - - @Override - protected void recordClientExecutionTime(Duration duration) - { - stats.addAwsClientExecuteTime(duration); - } - - @Override - protected void recordRetryPauseTime(Duration duration) - { - stats.addAwsClientRetryPauseTime(duration); - } - - @Override - protected void recordHttpClientPoolAvailableCount(long count) - { - stats.setAwsHttpClientPoolAvailableCount(count); - } - - @Override - protected void recordHttpClientPoolLeasedCount(long count) - { - stats.setAwsHttpClientPoolLeasedCount(count); - } - - @Override - protected void recordHttpClientPoolPendingCount(long count) - { - stats.setAwsHttpClientPoolPendingCount(count); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystemStats.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystemStats.java index 567e9ad57857..addedd59fce4 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystemStats.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystemStats.java @@ -14,17 +14,15 @@ package io.trino.plugin.hive.s3; import com.amazonaws.AbortedException; +import com.amazonaws.metrics.RequestMetricCollector; import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeStat; -import io.airlift.units.Duration; +import io.trino.plugin.hive.aws.AwsSdkClientCoreStats; +import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; import java.net.SocketException; import java.net.SocketTimeoutException; -import java.util.concurrent.atomic.AtomicLong; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; public class TrinoS3FileSystemStats { @@ -47,15 +45,7 @@ public class TrinoS3FileSystemStats private final CounterStat readRetries = new CounterStat(); // see AWSRequestMetrics - private final CounterStat awsRequestCount = new CounterStat(); - private final CounterStat awsRetryCount = new CounterStat(); - private final CounterStat awsThrottleExceptions = new CounterStat(); - private final TimeStat awsRequestTime = new TimeStat(MILLISECONDS); - private final TimeStat awsClientExecuteTime = new TimeStat(MILLISECONDS); - private final TimeStat awsClientRetryPauseTime = new TimeStat(MILLISECONDS); - private final AtomicLong awsHttpClientPoolAvailableCount = new AtomicLong(); - private final AtomicLong awsHttpClientPoolLeasedCount = new AtomicLong(); - private final AtomicLong awsHttpClientPoolPendingCount = new AtomicLong(); + private final AwsSdkClientCoreStats clientCoreStats = new AwsSdkClientCoreStats(); @Managed @Nested @@ -156,63 +146,10 @@ public CounterStat getAwsAbortedExceptions() } @Managed - @Nested - public CounterStat getAwsRequestCount() - { - return awsRequestCount; - } - - @Managed - @Nested - public CounterStat getAwsRetryCount() - { - return awsRetryCount; - } - - @Managed - @Nested - public CounterStat getAwsThrottleExceptions() + @Flatten + public AwsSdkClientCoreStats getClientCoreStats() { - return awsThrottleExceptions; - } - - @Managed - @Nested - public TimeStat getAwsRequestTime() - { - return awsRequestTime; - } - - @Managed - @Nested - public TimeStat getAwsClientExecuteTime() - { - return awsClientExecuteTime; - } - - @Managed - @Nested - public TimeStat getAwsClientRetryPauseTime() - { - return awsClientRetryPauseTime; - } - - @Managed - public long getAwsHttpClientPoolAvailableCount() - { - return awsHttpClientPoolAvailableCount.get(); - } - - @Managed - public long getAwsHttpClientPoolLeasedCount() - { - return awsHttpClientPoolLeasedCount.get(); - } - - @Managed - public long getAwsHttpClientPoolPendingCount() - { - return awsHttpClientPoolPendingCount.get(); + return clientCoreStats; } @Managed @@ -236,6 +173,11 @@ public CounterStat getReadRetries() return readRetries; } + public RequestMetricCollector newRequestMetricCollector() + { + return clientCoreStats.newRequestMetricCollector(); + } + public void connectionOpened() { activeConnections.update(1); @@ -307,51 +249,6 @@ public void newGetMetadataError() getMetadataErrors.update(1); } - public void updateAwsRequestCount(long requestCount) - { - awsRequestCount.update(requestCount); - } - - public void updateAwsRetryCount(long retryCount) - { - awsRetryCount.update(retryCount); - } - - public void updateAwsThrottleExceptionsCount(long throttleExceptionsCount) - { - awsThrottleExceptions.update(throttleExceptionsCount); - } - - public void addAwsRequestTime(Duration duration) - { - awsRequestTime.add(duration); - } - - public void addAwsClientExecuteTime(Duration duration) - { - awsClientExecuteTime.add(duration); - } - - public void addAwsClientRetryPauseTime(Duration duration) - { - awsClientRetryPauseTime.add(duration); - } - - public void setAwsHttpClientPoolAvailableCount(long count) - { - this.awsHttpClientPoolAvailableCount.set(count); - } - - public void setAwsHttpClientPoolLeasedCount(long count) - { - this.awsHttpClientPoolLeasedCount.set(count); - } - - public void setAwsHttpClientPoolPendingCount(long count) - { - this.awsHttpClientPoolPendingCount.set(count); - } - public void newGetObjectRetry() { getObjectRetries.update(1); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3select/TrinoS3ClientFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3select/TrinoS3ClientFactory.java index 729278be4da5..a179360e1456 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3select/TrinoS3ClientFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3select/TrinoS3ClientFactory.java @@ -27,7 +27,6 @@ import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.s3.HiveS3Config; import io.trino.plugin.hive.s3.TrinoS3FileSystem; -import io.trino.plugin.hive.s3.TrinoS3FileSystemMetricCollector; import org.apache.hadoop.conf.Configuration; import javax.annotation.concurrent.GuardedBy; @@ -109,7 +108,7 @@ private AmazonS3 createS3Client(Configuration config) AmazonS3Builder, ? extends AmazonS3> clientBuilder = AmazonS3Client.builder() .withCredentials(awsCredentialsProvider) .withClientConfiguration(clientConfiguration) - .withMetricsCollector(new TrinoS3FileSystemMetricCollector(TrinoS3FileSystem.getFileSystemStats())) + .withMetricsCollector(TrinoS3FileSystem.getFileSystemStats().newRequestMetricCollector()) .enablePathStyleAccess(); boolean regionOrEndpointSet = false; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/GlueMetastoreMethod.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/GlueMetastoreMethod.java index 283b2b1ad3c2..b9ef1975c10f 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/GlueMetastoreMethod.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/GlueMetastoreMethod.java @@ -14,7 +14,7 @@ package io.trino.plugin.iceberg.catalog.glue; import com.google.common.math.DoubleMath; -import io.trino.plugin.hive.metastore.glue.GlueMetastoreApiStats; +import io.trino.plugin.hive.aws.AwsApiCallStats; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; import java.math.RoundingMode; @@ -51,14 +51,14 @@ public enum GlueMetastoreMethod UPDATE_TABLE(GlueMetastoreStats::getUpdateTable), /**/; - private final Function extractor; + private final Function extractor; - GlueMetastoreMethod(Function extractor) + GlueMetastoreMethod(Function extractor) { this.extractor = requireNonNull(extractor, "extractor is null"); } - public GlueMetastoreApiStats getStatFrom(GlueMetastoreStats stats) + public AwsApiCallStats getStatFrom(GlueMetastoreStats stats) { return this.extractor.apply(stats); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java index 72fb45cb935f..b10ee5168963 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConnectorSmokeTest.java @@ -20,7 +20,7 @@ import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.collect.ImmutableMap; -import io.trino.plugin.hive.metastore.glue.GlueMetastoreApiStats; +import io.trino.plugin.hive.aws.AwsApiCallStats; import io.trino.plugin.iceberg.BaseIcebergConnectorSmokeTest; import io.trino.plugin.iceberg.IcebergQueryRunner; import io.trino.plugin.iceberg.SchemaInitializer; @@ -95,7 +95,7 @@ public void cleanup() listObjectsRequest, ListObjectsV2Request::setContinuationToken, ListObjectsV2Result::getNextContinuationToken, - new GlueMetastoreApiStats()) + new AwsApiCallStats()) .map(ListObjectsV2Result::getObjectSummaries) .flatMap(objectSummaries -> objectSummaries.stream().map(S3ObjectSummary::getKey)) .map(DeleteObjectsRequest.KeyVersion::new) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogMaterializedViewTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogMaterializedViewTest.java index f86b5368ff0b..9d39e4f3428d 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogMaterializedViewTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogMaterializedViewTest.java @@ -22,7 +22,7 @@ import com.amazonaws.services.glue.model.Table; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.trino.plugin.hive.metastore.glue.GlueMetastoreApiStats; +import io.trino.plugin.hive.aws.AwsApiCallStats; import io.trino.plugin.iceberg.BaseIcebergMaterializedViewTest; import io.trino.plugin.iceberg.IcebergQueryRunner; import io.trino.plugin.iceberg.SchemaInitializer; @@ -92,7 +92,7 @@ private static void cleanUpSchema(String schema) new GetTablesRequest().withDatabaseName(schema), GetTablesRequest::setNextToken, GetTablesResult::getNextToken, - new GlueMetastoreApiStats()) + new AwsApiCallStats()) .map(GetTablesResult::getTableList) .flatMap(Collection::stream) .map(Table::getName)