Skip to content

Fix JSON serialization of QueryCompletedEvent #14604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
electrum merged 2 commits into from
Oct 13, 2022
Merged

Fix JSON serialization of QueryCompletedEvent #14604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1447ec2
Remove outdated SPI revapi exclusions
electrum Oct 12, 2022
4520bed
Revert "Expose stage buffer utilization distribution to event listener"
electrum Oct 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 0 additions & 25 deletions core/trino-main/src/main/java/io/trino/event/QueryMonitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,6 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur
true,
ImmutableList.of(),
ImmutableList.of(),
ImmutableList.of(),
Optional.empty()),
createQueryContext(
queryInfo.getSession(),
Expand Down Expand Up @@ -325,7 +324,6 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo)
queryStats.getCompletedDrivers(),
queryInfo.isFinalQueryInfo(),
getCpuDistributions(queryInfo),
getStageOutputBufferUtilizationDistributions(queryInfo),
operatorSummaries.build(),
serializedPlanNodeStatsAndCosts);
}
Expand Down Expand Up @@ -705,29 +703,6 @@ private static StageCpuDistribution computeCpuDistribution(StageInfo stageInfo)
snapshot.getTotal() / snapshot.getCount());
}

private static List<Optional<io.trino.spi.metrics.Distribution<?>>> getStageOutputBufferUtilizationDistributions(QueryInfo queryInfo)
{
if (queryInfo.getOutputStage().isEmpty()) {
return ImmutableList.of();
}

ImmutableList.Builder<Optional<io.trino.spi.metrics.Distribution<?>>> builder = ImmutableList.builder();
populateStageOutputBufferUtilizationDistribution(queryInfo.getOutputStage().get(), builder);

return builder.build();
}

private static void populateStageOutputBufferUtilizationDistribution(StageInfo stageInfo, ImmutableList.Builder<Optional<io.trino.spi.metrics.Distribution<?>>> distributions)
{
Optional<io.trino.spi.metrics.Distribution<?>> distribution = stageInfo.getStageStats().getOutputBufferUtilization()
// cast to ?
.map(dist -> dist);
distributions.add(distribution);
for (StageInfo subStage : stageInfo.getSubStages()) {
populateStageOutputBufferUtilizationDistribution(subStage, distributions);
}
}

private static class FragmentNode
{
private final PlanFragmentId fragmentId;
Expand Down
90 changes: 5 additions & 85 deletions core/trino-spi/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,96 +180,16 @@
</old>
</item>
<!-- Backwards incompatible changes since the previous release -->
<!-- Any exclusions below can be deleted after each release (last cleared after 396) -->
<!-- Any exclusions below can be deleted after each release -->
<item>
<code>java.method.removed</code>
<old>method io.trino.spi.type.TypeSignature io.trino.spi.type.VarcharType::getParametrizedVarcharSignature(java.lang.String)</old>
</item>
<item>
<ignore>true</ignore>
<code>java.annotation.removed</code>
<old>parameter void io.trino.spi.statistics.ColumnStatisticMetadata::&lt;init&gt;(===java.lang.String===, io.trino.spi.statistics.ColumnStatisticType)</old>
<new>parameter void io.trino.spi.statistics.ColumnStatisticMetadata::&lt;init&gt;(===java.lang.String===, io.trino.spi.statistics.ColumnStatisticType)</new>
<annotation>@com.fasterxml.jackson.annotation.JsonProperty("columnName")</annotation>
</item>
<item>
<ignore>true</ignore>
<code>java.annotation.removed</code>
<old>parameter void io.trino.spi.statistics.ColumnStatisticMetadata::&lt;init&gt;(java.lang.String, ===io.trino.spi.statistics.ColumnStatisticType===)</old>
<new>parameter void io.trino.spi.statistics.ColumnStatisticMetadata::&lt;init&gt;(java.lang.String, ===io.trino.spi.statistics.ColumnStatisticType===)</new>
<annotation>@com.fasterxml.jackson.annotation.JsonProperty("statisticType")</annotation>
</item>
<item>
<ignore>true</ignore>
<code>java.annotation.removed</code>
<old>method void io.trino.spi.statistics.ColumnStatisticMetadata::&lt;init&gt;(java.lang.String, io.trino.spi.statistics.ColumnStatisticType)</old>
<new>method void io.trino.spi.statistics.ColumnStatisticMetadata::&lt;init&gt;(java.lang.String, io.trino.spi.statistics.ColumnStatisticType)</new>
<annotation>@com.fasterxml.jackson.annotation.JsonCreator</annotation>
</item>
<item>
<ignore>true</ignore>
<code>java.annotation.removed</code>
<old>method io.trino.spi.statistics.ColumnStatisticType io.trino.spi.statistics.ColumnStatisticMetadata::getStatisticType()</old>
<new>method io.trino.spi.statistics.ColumnStatisticType io.trino.spi.statistics.ColumnStatisticMetadata::getStatisticType()</new>
<annotation>@com.fasterxml.jackson.annotation.JsonProperty</annotation>
</item>
<item>
<code>java.method.visibilityReduced</code>
<old>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, int, io.trino.spi.block.Block, int[], boolean, boolean, io.trino.spi.block.DictionaryId)</old>
<new>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, int, io.trino.spi.block.Block, int[], boolean, boolean, io.trino.spi.block.DictionaryId)</new>
<oldVisibility>public</oldVisibility>
<newVisibility>private</newVisibility>
<code>java.method.numberOfParametersChanged</code>
<old>method void io.trino.spi.eventlistener.QueryStatistics::&lt;init&gt;(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List&lt;io.trino.spi.eventlistener.StageGcStatistics&gt;, int, boolean, java.util.List&lt;io.trino.spi.eventlistener.StageCpuDistribution&gt;, java.util.List&lt;java.util.Optional&lt;io.trino.spi.metrics.Distribution&lt;?&gt;&gt;&gt;, java.util.List&lt;java.lang.String&gt;, java.util.Optional&lt;java.lang.String&gt;)</old>
<new>method void io.trino.spi.eventlistener.QueryStatistics::&lt;init&gt;(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List&lt;io.trino.spi.eventlistener.StageGcStatistics&gt;, int, boolean, java.util.List&lt;io.trino.spi.eventlistener.StageCpuDistribution&gt;, java.util.List&lt;java.lang.String&gt;, java.util.Optional&lt;java.lang.String&gt;)</new>
</item>
<item>
<ignore>true</ignore>
<code>java.method.visibilityReduced</code>
<old>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(io.trino.spi.block.Block, int[])</old>
<new>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, int, io.trino.spi.block.Block, int[])</new>
<oldVisibility>public</oldVisibility>
<newVisibility>package</newVisibility>
</item>
<item>
<code>java.method.removed</code>
<old>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, io.trino.spi.block.Block, int[])</old>
</item>
<item>
<code>java.method.removed</code>
<old>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, io.trino.spi.block.Block, int[], boolean)</old>
</item>
<item>
<code>java.method.removed</code>
<old>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, io.trino.spi.block.Block, int[], boolean, io.trino.spi.block.DictionaryId)</old>
</item>
<item>
<code>java.method.removed</code>
<old>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, int, io.trino.spi.block.Block, int[], boolean, io.trino.spi.block.DictionaryId)</old>
</item>
<item>
<code>java.method.removed</code>
<old>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, io.trino.spi.block.Block, int[], boolean, io.trino.spi.block.DictionaryId)</old>
</item>
<item>
<code>java.method.removed</code>
<old>method void io.trino.spi.block.DictionaryBlock::&lt;init&gt;(int, io.trino.spi.block.Block, int[], io.trino.spi.block.DictionaryId)</old>
</item>
<item>
<code>java.method.returnTypeChanged</code>
<old>method io.trino.spi.block.RunLengthEncodedBlock io.trino.spi.block.RunLengthBlockEncoding::readBlock(io.trino.spi.block.BlockEncodingSerde, io.airlift.slice.SliceInput)</old>
<new>method io.trino.spi.block.Block io.trino.spi.block.RunLengthBlockEncoding::readBlock(io.trino.spi.block.BlockEncodingSerde, io.airlift.slice.SliceInput)</new>
</item>
<item>
<code>java.method.visibilityReduced</code>
<old>method void io.trino.spi.block.RunLengthEncodedBlock::&lt;init&gt;(io.trino.spi.block.Block, int)</old>
<new>method void io.trino.spi.block.RunLengthEncodedBlock::&lt;init&gt;(io.trino.spi.block.Block, int)</new>
<oldVisibility>public</oldVisibility>
<newVisibility>private</newVisibility>
</item>
<item>
<ignore>true</ignore>
<code>java.method.numberOfParametersChanged</code>
<old>method void io.trino.spi.eventlistener.QueryStatistics::&lt;init&gt;(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List&lt;io.trino.spi.eventlistener.StageGcStatistics&gt;, int, boolean, java.util.List&lt;io.trino.spi.eventlistener.StageCpuDistribution&gt;, java.util.List&lt;java.lang.String&gt;, java.util.Optional&lt;java.lang.String&gt;)</old>
<new>method void io.trino.spi.eventlistener.QueryStatistics::&lt;init&gt;(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, java.util.Optional&lt;java.time.Duration&gt;, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List&lt;io.trino.spi.eventlistener.StageGcStatistics&gt;, int, boolean, java.util.List&lt;io.trino.spi.eventlistener.StageCpuDistribution&gt;, java.util.List&lt;java.util.Optional&lt;io.trino.spi.metrics.Distribution&lt;?&gt;&gt;&gt;, java.util.List&lt;java.lang.String&gt;, java.util.Optional&lt;java.lang.String&gt;)
</new>
<old>method java.util.List&lt;java.util.Optional&lt;io.trino.spi.metrics.Distribution&lt;?&gt;&gt;&gt; io.trino.spi.eventlistener.QueryStatistics::getStageOutputBufferUtilizationDistribution()</old>
</item>
</differences>
</revapi.differences>
Expand Down
10 changes: 0 additions & 10 deletions core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.trino.spi.metrics.Distribution;

import java.time.Duration;
import java.util.List;
Expand Down Expand Up @@ -68,7 +67,6 @@ public class QueryStatistics
private final boolean complete;

private final List<StageCpuDistribution> cpuTimeDistribution;
private final List<Optional<Distribution<?>>> stageOutputBufferUtilizationDistribution;

/**
* Operator summaries serialized to JSON. Serialization format and structure
Expand Down Expand Up @@ -118,7 +116,6 @@ public QueryStatistics(
int completedSplits,
boolean complete,
List<StageCpuDistribution> cpuTimeDistribution,
List<Optional<Distribution<?>>> stageOutputBufferUtilizationDistribution,
List<String> operatorSummaries,
Optional<String> planNodeStatsAndCosts)
{
Expand Down Expand Up @@ -157,7 +154,6 @@ public QueryStatistics(
this.completedSplits = completedSplits;
this.complete = complete;
this.cpuTimeDistribution = requireNonNull(cpuTimeDistribution, "cpuTimeDistribution is null");
this.stageOutputBufferUtilizationDistribution = requireNonNull(stageOutputBufferUtilizationDistribution, "stageOutputBufferUtilizationDistribution is null");
this.operatorSummaries = requireNonNull(operatorSummaries, "operatorSummaries is null");
this.planNodeStatsAndCosts = requireNonNull(planNodeStatsAndCosts, "planNodeStatsAndCosts is null");
}
Expand Down Expand Up @@ -372,12 +368,6 @@ public List<StageCpuDistribution> getCpuTimeDistribution()
return cpuTimeDistribution;
}

@JsonProperty
public List<Optional<Distribution<?>>> getStageOutputBufferUtilizationDistribution()
{
return stageOutputBufferUtilizationDistribution;
}

@JsonProperty
public List<String> getOperatorSummaries()
{
Expand Down
6 changes: 0 additions & 6 deletions plugin/trino-http-event-listener/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,6 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-plugin-toolkit</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-testing-services</artifactId>
Expand Down
2 changes: 0 additions & 2 deletions ...no-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpEventListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import com.google.common.collect.ImmutableMap;
import io.airlift.json.JsonCodec;
import io.trino.operator.RetryPolicy;
import io.trino.plugin.base.metrics.TDigestHistogram;
import io.trino.spi.eventlistener.EventListener;
import io.trino.spi.eventlistener.QueryCompletedEvent;
import io.trino.spi.eventlistener.QueryContext;
Expand Down Expand Up @@ -180,7 +179,6 @@ public class TestHttpEventListener
0,
true,
Collections.emptyList(),
List.of(Optional.of(TDigestHistogram.fromValue(42))),
Collections.emptyList(),
Optional.empty());

Expand Down
2 changes: 0 additions & 2 deletions testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerWithSplits.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,6 @@ public void testSplitsForNormalQuery()
assertTrue(statistics.getWallTime().getSeconds() >= 0);
assertTrue(statistics.getCpuTimeDistribution().size() > 0);
assertTrue(statistics.getOperatorSummaries().size() > 0);
assertTrue(statistics.getStageOutputBufferUtilizationDistribution().size() > 0);
assertTrue(statistics.getStageOutputBufferUtilizationDistribution().stream().allMatch(Optional::isPresent));
}

@Test
Expand Down