From bad3c4db5a78b3d764b1d7bb94e9d25a5e6ae130 Mon Sep 17 00:00:00 2001 From: Mateusz Gajewski Date: Mon, 10 Jan 2022 11:57:16 +0100 Subject: [PATCH 1/2] Improve null statistics handling in product tests --- .../launcher/env/StatisticsFetcher.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/StatisticsFetcher.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/StatisticsFetcher.java index 2da1c60c96ac..58da99707800 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/StatisticsFetcher.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/StatisticsFetcher.java @@ -14,6 +14,7 @@ package io.trino.tests.product.launcher.env; import com.github.dockerjava.api.async.ResultCallback; +import com.github.dockerjava.api.model.CpuUsageConfig; import com.github.dockerjava.api.model.MemoryStatsConfig; import com.github.dockerjava.api.model.StatisticNetworksConfig; import com.github.dockerjava.api.model.Statistics; @@ -24,14 +25,20 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Stream; +import static com.google.common.base.MoreObjects.firstNonNull; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.trino.tests.product.launcher.env.StatisticsFetcher.Stats.statisticsAreEmpty; import static java.lang.String.format; +import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; public class StatisticsFetcher @@ -90,8 +97,8 @@ private Stats toStats(Statistics statistics, Stats previousStats) } Stats stats = new Stats(); - stats.systemCpuUsage = statistics.getCpuStats().getSystemCpuUsage(); - stats.totalCpuUsage = statistics.getCpuStats().getCpuUsage().getTotalUsage(); + stats.systemCpuUsage = firstNonNull(statistics.getCpuStats().getSystemCpuUsage(), -1L); + stats.totalCpuUsage = Optional.ofNullable(statistics.getCpuStats().getCpuUsage()).map(CpuUsageConfig::getTotalUsage).orElse(-1L); stats.cpuUsagePerc = 0.0; if (previousStats.systemCpuUsage != -1 && previousStats.totalCpuUsage != -1) { @@ -99,21 +106,20 @@ private Stats toStats(Statistics statistics, Stats previousStats) double systemCpuDelta = stats.systemCpuUsage - previousStats.systemCpuUsage; if (usageCpuDelta > 0.0 && systemCpuDelta > 0.0) { - stats.cpuUsagePerc = usageCpuDelta / systemCpuDelta * statistics.getCpuStats().getCpuUsage().getPercpuUsage().size() * 100; + stats.cpuUsagePerc = usageCpuDelta / systemCpuDelta * Optional.ofNullable(statistics.getCpuStats().getCpuUsage().getPercpuUsage()).map(List::size).orElse(1) * 100; } } MemoryStatsConfig memoryStats = statistics.getMemoryStats(); - stats.memoryLimit = DataSize.ofBytes(memoryStats.getLimit()).to(GIGABYTE); - stats.memoryUsage = DataSize.ofBytes(memoryStats.getUsage()).to(GIGABYTE); - stats.memoryMaxUsage = DataSize.ofBytes(memoryStats.getMaxUsage()).to(GIGABYTE); - stats.memoryUsagePerc = 100.0 * memoryStats.getUsage() / memoryStats.getLimit(); - - stats.pids = statistics.getPidsStats().getCurrent(); - - Supplier> stream = () -> statistics.getNetworks().values().stream(); - stats.networkReceived = DataSize.ofBytes(stream.get().map(StatisticNetworksConfig::getRxBytes).reduce(0L, Long::sum)).succinct(); - stats.networkSent = DataSize.ofBytes(stream.get().map(StatisticNetworksConfig::getTxBytes).reduce(0L, Long::sum)).succinct(); + stats.memoryLimit = DataSize.ofBytes(firstNonNull(memoryStats.getLimit(), 0L)).to(GIGABYTE); + stats.memoryUsage = DataSize.ofBytes(firstNonNull(memoryStats.getUsage(), 0L)).to(GIGABYTE); + stats.memoryMaxUsage = DataSize.ofBytes(firstNonNull(memoryStats.getMaxUsage(), 0L)).to(GIGABYTE); + stats.memoryUsagePerc = 100.0 * firstNonNull(memoryStats.getUsage(), 0L) / firstNonNull(memoryStats.getLimit(), 1L); + stats.pids = firstNonNull(statistics.getPidsStats().getCurrent(), -1L); + + Supplier> stream = () -> Optional.ofNullable(statistics.getNetworks()).map(Map::values).orElse(emptyList()).stream(); + stats.networkReceived = DataSize.ofBytes(stream.get().map(StatisticNetworksConfig::getRxBytes).filter(Objects::nonNull).reduce(0L, Long::sum)).succinct(); + stats.networkSent = DataSize.ofBytes(stream.get().map(StatisticNetworksConfig::getTxBytes).filter(Objects::nonNull).reduce(0L, Long::sum)).succinct(); return stats; } From af206cd3b48137d6a5f4a7dd5cb51321a9e67b37 Mon Sep 17 00:00:00 2001 From: Mateusz Gajewski Date: Mon, 10 Jan 2022 12:09:08 +0100 Subject: [PATCH 2/2] Remove peak memory usage statistic This statistics is not really useful and most of the time Docker engine returns null for it --- .../trino/tests/product/launcher/env/StatisticsFetcher.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/StatisticsFetcher.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/StatisticsFetcher.java index 58da99707800..29f320dd94e9 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/StatisticsFetcher.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/StatisticsFetcher.java @@ -113,7 +113,6 @@ private Stats toStats(Statistics statistics, Stats previousStats) MemoryStatsConfig memoryStats = statistics.getMemoryStats(); stats.memoryLimit = DataSize.ofBytes(firstNonNull(memoryStats.getLimit(), 0L)).to(GIGABYTE); stats.memoryUsage = DataSize.ofBytes(firstNonNull(memoryStats.getUsage(), 0L)).to(GIGABYTE); - stats.memoryMaxUsage = DataSize.ofBytes(firstNonNull(memoryStats.getMaxUsage(), 0L)).to(GIGABYTE); stats.memoryUsagePerc = 100.0 * firstNonNull(memoryStats.getUsage(), 0L) / firstNonNull(memoryStats.getLimit(), 1L); stats.pids = firstNonNull(statistics.getPidsStats().getCurrent(), -1L); @@ -190,7 +189,6 @@ public static class Stats "mem", "max mem", "mem %", - "peak mem", "pids", "net in", "net out" @@ -202,7 +200,6 @@ public static class Stats private double memoryUsagePerc; private DataSize memoryUsage; private DataSize memoryLimit; - private DataSize memoryMaxUsage; private long pids; public DataSize networkReceived; public DataSize networkSent; @@ -224,7 +221,6 @@ public String[] toRow(String name) memoryLimit.toString(), memoryUsage.toString(), format("%.2f%%", memoryUsagePerc), - memoryMaxUsage.toString(), format("%d", pids), format("%s", networkReceived), format("%s", networkSent)