Skip to content

Commit 1824bee

Browse files
raunaqmorarkaraunaqmorarka
committed
Remove tracking of rawInputDataSize and rawInputPositions
rawInputDataSize is redundant with internalNetworkInputDataSize + physicalInputDataSize rawInputPositions is redundant with processedInputPositions
1 parent c470369 commit 1824bee

File tree

51 files changed

+83
-458
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+83
-458
lines changed

‎core/trino-main/src/main/java/io/trino/connector/system/TaskSystemTable.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,7 @@ public class TaskSystemTable
6363
.column("split_cpu_time_ms", BIGINT)
6464
.column("split_blocked_time_ms", BIGINT)
6565

66-
.column("raw_input_bytes", BIGINT)
67-
.column("raw_input_rows", BIGINT)
66+
.column("internal_network_input_bytes", BIGINT)
6867

6968
.column("processed_input_bytes", BIGINT)
7069
.column("processed_input_rows", BIGINT)
@@ -127,8 +126,7 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect
127126
toMillis(stats.getTotalCpuTime()),
128127
toMillis(stats.getTotalBlockedTime()),
129128

130-
toBytes(stats.getRawInputDataSize()),
131-
stats.getRawInputPositions(),
129+
toBytes(stats.getInternalNetworkInputDataSize()),
132130

133131
toBytes(stats.getProcessedInputDataSize()),
134132
stats.getProcessedInputPositions(),

‎core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -324,10 +324,6 @@ private static QueryStats immediateFailureQueryStats()
324324
DataSize.ofBytes(0),
325325
0,
326326
0,
327-
DataSize.ofBytes(0),
328-
DataSize.ofBytes(0),
329-
0,
330-
0,
331327
new Duration(0, MILLISECONDS),
332328
new Duration(0, MILLISECONDS),
333329
DataSize.ofBytes(0),

‎core/trino-main/src/main/java/io/trino/event/QueryMonitor.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -231,8 +231,6 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur
231231
0,
232232
0,
233233
0,
234-
0,
235-
0,
236234
ImmutableList.of(),
237235
0,
238236
true,
@@ -342,8 +340,6 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo)
342340
queryStats.getProcessedInputPositions(),
343341
queryStats.getInternalNetworkInputDataSize().toBytes(),
344342
queryStats.getInternalNetworkInputPositions(),
345-
queryStats.getRawInputDataSize().toBytes(),
346-
queryStats.getRawInputPositions(),
347343
queryStats.getOutputDataSize().toBytes(),
348344
queryStats.getOutputPositions(),
349345
queryStats.getLogicalWrittenDataSize().toBytes(),
@@ -865,8 +861,8 @@ private StageTaskStatistics computeStageTaskStatistics(QueryInfo queryInfo, Stag
865861
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getTotalScheduledTime().toMillis())),
866862
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getPeakUserMemoryReservation().toBytes())),
867863
getTasksDistribution(stageInfo, taskInfo -> taskInfo.estimatedMemory().map(DataSize::toBytes)),
868-
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getRawInputDataSize().toBytes())),
869-
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getRawInputPositions())),
864+
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getInternalNetworkInputDataSize().toBytes() + taskInfo.stats().getPhysicalInputDataSize().toBytes())),
865+
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getProcessedInputPositions())),
870866
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getProcessedInputDataSize().toBytes())),
871867
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getProcessedInputPositions())),
872868
getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getOutputDataSize().toBytes())),

‎core/trino-main/src/main/java/io/trino/execution/BasicStageStats.java‎

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ public class BasicStageStats
4747

4848
DataSize.ofBytes(0),
4949
0,
50-
DataSize.ofBytes(0),
5150

5251
0,
52+
5353
DataSize.ofBytes(0),
5454

5555
0,
@@ -81,8 +81,7 @@ public class BasicStageStats
8181
private final DataSize physicalWrittenDataSize;
8282
private final DataSize internalNetworkInputDataSize;
8383
private final long internalNetworkInputPositions;
84-
private final DataSize rawInputDataSize;
85-
private final long rawInputPositions;
84+
private final long processedInputPositions;
8685
private final DataSize spilledDataSize;
8786
private final double cumulativeUserMemory;
8887
private final double failedCumulativeUserMemory;
@@ -116,8 +115,8 @@ public BasicStageStats(
116115
DataSize internalNetworkInputDataSize,
117116
long internalNetworkInputPositions,
118117

119-
DataSize rawInputDataSize,
120-
long rawInputPositions,
118+
long processedInputPositions,
119+
121120
DataSize spilledDataSize,
122121

123122
double cumulativeUserMemory,
@@ -149,8 +148,7 @@ public BasicStageStats(
149148
this.physicalWrittenDataSize = requireNonNull(physicalWrittenDataSize, "physicalWrittenDataSize is null");
150149
this.internalNetworkInputDataSize = requireNonNull(internalNetworkInputDataSize, "internalNetworkInputDataSize is null");
151150
this.internalNetworkInputPositions = internalNetworkInputPositions;
152-
this.rawInputDataSize = requireNonNull(rawInputDataSize, "rawInputDataSize is null");
153-
this.rawInputPositions = rawInputPositions;
151+
this.processedInputPositions = processedInputPositions;
154152
this.spilledDataSize = requireNonNull(spilledDataSize, "spilledDataSize is null");
155153
this.cumulativeUserMemory = cumulativeUserMemory;
156154
this.failedCumulativeUserMemory = failedCumulativeUserMemory;
@@ -231,14 +229,9 @@ public long getInternalNetworkInputPositions()
231229
return internalNetworkInputPositions;
232230
}
233231

234-
public DataSize getRawInputDataSize()
232+
public long getProcessedInputPositions()
235233
{
236-
return rawInputDataSize;
237-
}
238-
239-
public long getRawInputPositions()
240-
{
241-
return rawInputPositions;
234+
return processedInputPositions;
242235
}
243236

244237
public DataSize getSpilledDataSize()
@@ -334,8 +327,7 @@ public static BasicStageStats aggregateBasicStageStats(Iterable<BasicStageStats>
334327
long internalNetworkInputDataSize = 0;
335328
long internalNetworkInputPositions = 0;
336329

337-
long rawInputDataSize = 0;
338-
long rawInputPositions = 0;
330+
long processedInputPositions = 0;
339331
long spilledDataSize = 0;
340332

341333
boolean isScheduled = true;
@@ -375,8 +367,7 @@ public static BasicStageStats aggregateBasicStageStats(Iterable<BasicStageStats>
375367
internalNetworkInputDataSize += stageStats.getInternalNetworkInputDataSize().toBytes();
376368
internalNetworkInputPositions += stageStats.getInternalNetworkInputPositions();
377369

378-
rawInputDataSize += stageStats.getRawInputDataSize().toBytes();
379-
rawInputPositions += stageStats.getRawInputPositions();
370+
processedInputPositions += stageStats.getProcessedInputPositions();
380371
spilledDataSize += stageStats.getSpilledDataSize().toBytes();
381372
}
382373

@@ -408,8 +399,8 @@ public static BasicStageStats aggregateBasicStageStats(Iterable<BasicStageStats>
408399
succinctBytes(internalNetworkInputDataSize),
409400
internalNetworkInputPositions,
410401

411-
succinctBytes(rawInputDataSize),
412-
rawInputPositions,
402+
processedInputPositions,
403+
413404
succinctBytes(spilledDataSize),
414405

415406
cumulativeUserMemory,

‎core/trino-main/src/main/java/io/trino/execution/QueryManagerStats.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,11 @@ private void queryFinished(BasicQueryInfo info)
7373
{
7474
completedQueries.update(1);
7575

76-
long rawInputBytes = info.getQueryStats().getRawInputDataSize().toBytes();
76+
long rawInputBytes = info.getQueryStats().getPhysicalInputDataSize().toBytes() + info.getQueryStats().getInternalNetworkInputDataSize().toBytes();
7777

7878
consumedCpuTimeSecs.update((long) info.getQueryStats().getTotalCpuTime().getValue(SECONDS));
79-
consumedInputBytes.update(info.getQueryStats().getRawInputDataSize().toBytes());
80-
consumedInputRows.update(info.getQueryStats().getRawInputPositions());
79+
consumedInputBytes.update(rawInputBytes);
80+
consumedInputRows.update(info.getQueryStats().getProcessedInputPositions());
8181
executionTime.add(info.getQueryStats().getExecutionTime());
8282
queuedTime.add(info.getQueryStats().getQueuedTime());
8383

‎core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java‎

Lines changed: 1 addition & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -556,8 +556,7 @@ private BasicQueryStats createBasicQueryStats(BasicStageStats stageStats)
556556
stageStats.getCompletedDrivers(),
557557
stageStats.getBlockedDrivers(),
558558

559-
stageStats.getRawInputDataSize(),
560-
stageStats.getRawInputPositions(),
559+
stageStats.getProcessedInputPositions(),
561560
stageStats.getSpilledDataSize(),
562561
stageStats.getPhysicalInputDataSize(),
563562
stageStats.getPhysicalWrittenDataSize(),
@@ -689,11 +688,6 @@ private QueryStats getQueryStats(Optional<StagesInfo> stages)
689688
long internalNetworkInputPositions = 0;
690689
long failedInternalNetworkInputPositions = 0;
691690

692-
long rawInputDataSize = 0;
693-
long failedRawInputDataSize = 0;
694-
long rawInputPositions = 0;
695-
long failedRawInputPositions = 0;
696-
697691
long processedInputDataSize = 0;
698692
long failedProcessedInputDataSize = 0;
699693
long processedInputPositions = 0;
@@ -762,11 +756,6 @@ private QueryStats getQueryStats(Optional<StagesInfo> stages)
762756

763757
PlanFragment plan = stageInfo.getPlan();
764758
if (plan != null && plan.containsTableScanNode()) {
765-
rawInputDataSize += stageStats.getRawInputDataSize().toBytes();
766-
failedRawInputDataSize += stageStats.getFailedRawInputDataSize().toBytes();
767-
rawInputPositions += stageStats.getRawInputPositions();
768-
failedRawInputPositions += stageStats.getFailedRawInputPositions();
769-
770759
processedInputDataSize += stageStats.getProcessedInputDataSize().toBytes();
771760
failedProcessedInputDataSize += stageStats.getFailedProcessedInputDataSize().toBytes();
772761
processedInputPositions += stageStats.getProcessedInputPositions();
@@ -916,10 +905,6 @@ private QueryStats getQueryStats(Optional<StagesInfo> stages)
916905
succinctBytes(failedInternalNetworkInputDataSize),
917906
internalNetworkInputPositions,
918907
failedInternalNetworkInputPositions,
919-
succinctBytes(rawInputDataSize),
920-
succinctBytes(failedRawInputDataSize),
921-
rawInputPositions,
922-
failedRawInputPositions,
923908
succinctBytes(processedInputDataSize),
924909
succinctBytes(failedProcessedInputDataSize),
925910
processedInputPositions,
@@ -1538,10 +1523,6 @@ private static QueryStats pruneQueryStats(QueryStats queryStats)
15381523
queryStats.getFailedInternalNetworkInputDataSize(),
15391524
queryStats.getInternalNetworkInputPositions(),
15401525
queryStats.getFailedInternalNetworkInputPositions(),
1541-
queryStats.getRawInputDataSize(),
1542-
queryStats.getFailedRawInputDataSize(),
1543-
queryStats.getRawInputPositions(),
1544-
queryStats.getFailedRawInputPositions(),
15451526
queryStats.getProcessedInputDataSize(),
15461527
queryStats.getFailedProcessedInputDataSize(),
15471528
queryStats.getProcessedInputPositions(),

‎core/trino-main/src/main/java/io/trino/execution/QueryStats.java‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -104,11 +104,6 @@ public class QueryStats
104104
private final long internalNetworkInputPositions;
105105
private final long failedInternalNetworkInputPositions;
106106

107-
private final DataSize rawInputDataSize;
108-
private final DataSize failedRawInputDataSize;
109-
private final long rawInputPositions;
110-
private final long failedRawInputPositions;
111-
112107
private final DataSize processedInputDataSize;
113108
private final DataSize failedProcessedInputDataSize;
114109
private final long processedInputPositions;
@@ -201,11 +196,6 @@ public QueryStats(
201196
@JsonProperty("internalNetworkInputPositions") long internalNetworkInputPositions,
202197
@JsonProperty("failedInternalNetworkInputPositions") long failedInternalNetworkInputPositions,
203198

204-
@JsonProperty("rawInputDataSize") DataSize rawInputDataSize,
205-
@JsonProperty("failedRawInputDataSize") DataSize failedRawInputDataSize,
206-
@JsonProperty("rawInputPositions") long rawInputPositions,
207-
@JsonProperty("failedRawInputPositions") long failedRawInputPositions,
208-
209199
@JsonProperty("processedInputDataSize") DataSize processedInputDataSize,
210200
@JsonProperty("failedProcessedInputDataSize") DataSize failedProcessedInputDataSize,
211201
@JsonProperty("processedInputPositions") long processedInputPositions,
@@ -307,13 +297,6 @@ public QueryStats(
307297
checkArgument(failedInternalNetworkInputPositions >= 0, "failedInternalNetworkInputPositions is negative");
308298
this.failedInternalNetworkInputPositions = failedInternalNetworkInputPositions;
309299

310-
this.rawInputDataSize = requireNonNull(rawInputDataSize, "rawInputDataSize is null");
311-
this.failedRawInputDataSize = requireNonNull(failedRawInputDataSize, "failedRawInputDataSize is null");
312-
checkArgument(rawInputPositions >= 0, "rawInputPositions is negative");
313-
this.rawInputPositions = rawInputPositions;
314-
checkArgument(failedRawInputPositions >= 0, "failedRawInputPositions is negative");
315-
this.failedRawInputPositions = failedRawInputPositions;
316-
317300
this.processedInputDataSize = requireNonNull(processedInputDataSize, "processedInputDataSize is null");
318301
this.failedProcessedInputDataSize = requireNonNull(failedProcessedInputDataSize, "failedProcessedInputDataSize is null");
319302
checkArgument(processedInputPositions >= 0, "processedInputPositions is negative");
@@ -672,30 +655,6 @@ public long getFailedInternalNetworkInputPositions()
672655
return failedInternalNetworkInputPositions;
673656
}
674657

675-
@JsonProperty
676-
public DataSize getRawInputDataSize()
677-
{
678-
return rawInputDataSize;
679-
}
680-
681-
@JsonProperty
682-
public DataSize getFailedRawInputDataSize()
683-
{
684-
return failedRawInputDataSize;
685-
}
686-
687-
@JsonProperty
688-
public long getRawInputPositions()
689-
{
690-
return rawInputPositions;
691-
}
692-
693-
@JsonProperty
694-
public long getFailedRawInputPositions()
695-
{
696-
return failedRawInputPositions;
697-
}
698-
699658
@JsonProperty
700659
public DataSize getProcessedInputDataSize()
701660
{

‎core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java‎

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -294,8 +294,8 @@ public BasicStageStats getBasicStageStats(Supplier<Iterable<TaskInfo>> taskInfos
294294
long internalNetworkInputDataSize = 0;
295295
long internalNetworkInputPositions = 0;
296296

297-
long rawInputDataSize = 0;
298-
long rawInputPositions = 0;
297+
long processedInputPositions = 0;
298+
299299
long spilledDataSize = 0;
300300

301301
boolean fullyBlocked = true;
@@ -346,10 +346,7 @@ public BasicStageStats getBasicStageStats(Supplier<Iterable<TaskInfo>> taskInfos
346346
internalNetworkInputDataSize += taskStats.getInternalNetworkInputDataSize().toBytes();
347347
internalNetworkInputPositions += taskStats.getInternalNetworkInputPositions();
348348

349-
if (fragment.containsTableScanNode()) {
350-
rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
351-
rawInputPositions += taskStats.getRawInputPositions();
352-
}
349+
processedInputPositions += taskStats.getProcessedInputPositions();
353350

354351
spilledDataSize += taskStats.getSpilledDataSize().toBytes();
355352
}
@@ -382,8 +379,8 @@ public BasicStageStats getBasicStageStats(Supplier<Iterable<TaskInfo>> taskInfos
382379
succinctBytes(internalNetworkInputDataSize),
383380
internalNetworkInputPositions,
384381

385-
succinctBytes(rawInputDataSize),
386-
rawInputPositions,
382+
processedInputPositions,
383+
387384
succinctBytes(spilledDataSize),
388385

389386
cumulativeUserMemory,
@@ -457,11 +454,6 @@ public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier)
457454
long internalNetworkInputPositions = 0;
458455
long failedInternalNetworkInputPositions = 0;
459456

460-
long rawInputDataSize = 0;
461-
long failedRawInputDataSize = 0;
462-
long rawInputPositions = 0;
463-
long failedRawInputPositions = 0;
464-
465457
long processedInputDataSize = 0;
466458
long failedProcessedInputDataSize = 0;
467459
long processedInputPositions = 0;
@@ -542,9 +534,6 @@ public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier)
542534
internalNetworkInputDataSize += taskStats.getInternalNetworkInputDataSize().toBytes();
543535
internalNetworkInputPositions += taskStats.getInternalNetworkInputPositions();
544536

545-
rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
546-
rawInputPositions += taskStats.getRawInputPositions();
547-
548537
processedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
549538
processedInputPositions += taskStats.getProcessedInputPositions();
550539

@@ -571,9 +560,6 @@ public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier)
571560
failedInternalNetworkInputDataSize += taskStats.getInternalNetworkInputDataSize().toBytes();
572561
failedInternalNetworkInputPositions += taskStats.getInternalNetworkInputPositions();
573562

574-
failedRawInputDataSize += taskStats.getRawInputDataSize().toBytes();
575-
failedRawInputPositions += taskStats.getRawInputPositions();
576-
577563
failedProcessedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
578564
failedProcessedInputPositions += taskStats.getProcessedInputPositions();
579565

@@ -650,11 +636,6 @@ public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier)
650636
internalNetworkInputPositions,
651637
failedInternalNetworkInputPositions,
652638

653-
succinctBytes(rawInputDataSize),
654-
succinctBytes(failedRawInputDataSize),
655-
rawInputPositions,
656-
failedRawInputPositions,
657-
658639
succinctBytes(processedInputDataSize),
659640
succinctBytes(failedProcessedInputDataSize),
660641
processedInputPositions,

0 commit comments

Comments
 (0)