diff --git a/core/trino-main/src/main/java/io/trino/ExceededMemoryLimitException.java b/core/trino-main/src/main/java/io/trino/ExceededMemoryLimitException.java index 30264d8d842b..b0f57af45bcd 100644 --- a/core/trino-main/src/main/java/io/trino/ExceededMemoryLimitException.java +++ b/core/trino-main/src/main/java/io/trino/ExceededMemoryLimitException.java @@ -37,19 +37,13 @@ public static ExceededMemoryLimitException exceededGlobalTotalLimit(DataSize max public static ExceededMemoryLimitException exceededLocalUserMemoryLimit(DataSize maxMemory, String additionalFailureInfo) { return new ExceededMemoryLimitException(EXCEEDED_LOCAL_MEMORY_LIMIT, - format("Query exceeded per-node user memory limit of %s [%s]", maxMemory, additionalFailureInfo)); - } - - public static ExceededMemoryLimitException exceededLocalTotalMemoryLimit(DataSize maxMemory, String additionalFailureInfo) - { - return new ExceededMemoryLimitException(EXCEEDED_LOCAL_MEMORY_LIMIT, - format("Query exceeded per-node total memory limit of %s [%s]", maxMemory, additionalFailureInfo)); + format("Query exceeded per-node memory limit of %s [%s]", maxMemory, additionalFailureInfo)); } public static ExceededMemoryLimitException exceededTaskMemoryLimit(DataSize maxMemory, String additionalFailureInfo) { return new ExceededMemoryLimitException(EXCEEDED_LOCAL_MEMORY_LIMIT, - format("Query exceeded per-task total memory limit of %s [%s]", maxMemory, additionalFailureInfo)); + format("Query exceeded per-task memory limit of %s [%s]", maxMemory, additionalFailureInfo)); } private ExceededMemoryLimitException(StandardErrorCode errorCode, String message) diff --git a/core/trino-main/src/main/java/io/trino/SystemSessionProperties.java b/core/trino-main/src/main/java/io/trino/SystemSessionProperties.java index fd89670c855c..eef590c6c3af 100644 --- a/core/trino-main/src/main/java/io/trino/SystemSessionProperties.java +++ b/core/trino-main/src/main/java/io/trino/SystemSessionProperties.java @@ -128,8 +128,7 @@ public final class SystemSessionProperties public static final String ENABLE_COORDINATOR_DYNAMIC_FILTERS_DISTRIBUTION = "enable_coordinator_dynamic_filters_distribution"; public static final String ENABLE_LARGE_DYNAMIC_FILTERS = "enable_large_dynamic_filters"; public static final String QUERY_MAX_MEMORY_PER_NODE = "query_max_memory_per_node"; - public static final String QUERY_MAX_TOTAL_MEMORY_PER_NODE = "query_max_total_memory_per_node"; - public static final String QUERY_MAX_TOTAL_MEMORY_PER_TASK = "query_max_total_memory_per_task"; + public static final String QUERY_MAX_MEMORY_PER_TASK = "query_max_memory_per_task"; public static final String IGNORE_DOWNSTREAM_PREFERENCES = "ignore_downstream_preferences"; public static final String FILTERING_SEMI_JOIN_TO_INNER = "rewrite_filtering_semi_join_to_inner_join"; public static final String OPTIMIZE_DUPLICATE_INSENSITIVE_JOINS = "optimize_duplicate_insensitive_joins"; @@ -581,14 +580,9 @@ public SystemSessionProperties( nodeMemoryConfig.getMaxQueryMemoryPerNode(), true), dataSizeProperty( - QUERY_MAX_TOTAL_MEMORY_PER_NODE, - "Maximum amount of total memory a query can use per node", - nodeMemoryConfig.getMaxQueryTotalMemoryPerNode(), - true), - dataSizeProperty( - QUERY_MAX_TOTAL_MEMORY_PER_TASK, + QUERY_MAX_MEMORY_PER_TASK, "Maximum amount of memory a single task can use", - nodeMemoryConfig.getMaxQueryTotalMemoryPerTask().orElse(null), + nodeMemoryConfig.getMaxQueryMemoryPerTask().orElse(null), true), booleanProperty( IGNORE_DOWNSTREAM_PREFERENCES, @@ -1151,14 +1145,9 @@ public static DataSize getQueryMaxMemoryPerNode(Session session) return session.getSystemProperty(QUERY_MAX_MEMORY_PER_NODE, DataSize.class); } - public static DataSize getQueryMaxTotalMemoryPerNode(Session session) - { - return session.getSystemProperty(QUERY_MAX_TOTAL_MEMORY_PER_NODE, DataSize.class); - } - public static Optional getQueryMaxTotalMemoryPerTask(Session session) { - return Optional.ofNullable(session.getSystemProperty(QUERY_MAX_TOTAL_MEMORY_PER_TASK, DataSize.class)); + return Optional.ofNullable(session.getSystemProperty(QUERY_MAX_MEMORY_PER_TASK, DataSize.class)); } public static boolean ignoreDownStreamPreferences(Session session) diff --git a/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java b/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java index 02c611526e1f..bb9da7432f8a 100644 --- a/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java +++ b/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java @@ -196,7 +196,7 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return memoryUsageBytes + pageBuilder.getRetainedSizeInBytes(); } diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java b/core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java index 83b01967d12d..4848390a4cea 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java @@ -272,8 +272,6 @@ private static QueryStats immediateFailureQueryStats() 0, 0, 0, - 0, - DataSize.ofBytes(0), DataSize.ofBytes(0), DataSize.ofBytes(0), DataSize.ofBytes(0), diff --git a/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java b/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java index 878e8b08715c..a9cecf3b3c9b 100644 --- a/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java +++ b/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java @@ -189,8 +189,6 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur 0, 0, 0, - 0, - 0, ImmutableList.of(), 0, true, @@ -263,7 +261,6 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo) Optional.of(ofMillis(queryStats.getPlanningTime().toMillis())), Optional.of(ofMillis(queryStats.getExecutionTime().toMillis())), queryStats.getPeakUserMemoryReservation().toBytes(), - queryStats.getPeakNonRevocableMemoryReservation().toBytes(), queryStats.getPeakTaskUserMemory().toBytes(), queryStats.getPeakTaskTotalMemory().toBytes(), queryStats.getPhysicalInputDataSize().toBytes(), @@ -277,7 +274,6 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo) queryStats.getLogicalWrittenDataSize().toBytes(), queryStats.getWrittenPositions(), queryStats.getCumulativeUserMemory(), - queryStats.getCumulativeSystemMemory(), queryStats.getStageGcStatistics(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), diff --git a/core/trino-main/src/main/java/io/trino/execution/BasicStageStats.java b/core/trino-main/src/main/java/io/trino/execution/BasicStageStats.java index 40242291851a..5b1ffc92de28 100644 --- a/core/trino-main/src/main/java/io/trino/execution/BasicStageStats.java +++ b/core/trino-main/src/main/java/io/trino/execution/BasicStageStats.java @@ -47,7 +47,6 @@ public class BasicStageStats DataSize.ofBytes(0), 0, - 0, 0, DataSize.ofBytes(0), DataSize.ofBytes(0), @@ -73,7 +72,6 @@ public class BasicStageStats private final DataSize rawInputDataSize; private final long rawInputPositions; private final long cumulativeUserMemory; - private final long cumulativeSystemMemory; private final DataSize userMemoryReservation; private final DataSize totalMemoryReservation; private final Duration totalCpuTime; @@ -101,7 +99,6 @@ public BasicStageStats( long rawInputPositions, long cumulativeUserMemory, - long cumulativeSystemMemory, DataSize userMemoryReservation, DataSize totalMemoryReservation, @@ -126,7 +123,6 @@ public BasicStageStats( this.rawInputDataSize = requireNonNull(rawInputDataSize, "rawInputDataSize is null"); this.rawInputPositions = rawInputPositions; this.cumulativeUserMemory = cumulativeUserMemory; - this.cumulativeSystemMemory = cumulativeSystemMemory; this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null"); this.totalMemoryReservation = requireNonNull(totalMemoryReservation, "totalMemoryReservation is null"); this.totalCpuTime = requireNonNull(totalCpuTime, "totalCpuTime is null"); @@ -201,11 +197,6 @@ public long getCumulativeUserMemory() return cumulativeUserMemory; } - public long getCumulativeSystemMemory() - { - return cumulativeSystemMemory; - } - public DataSize getUserMemoryReservation() { return userMemoryReservation; @@ -249,7 +240,6 @@ public static BasicStageStats aggregateBasicStageStats(Iterable int completedDrivers = 0; long cumulativeUserMemory = 0; - long cumulativeSystemMemory = 0; long userMemoryReservation = 0; long totalMemoryReservation = 0; @@ -278,7 +268,6 @@ public static BasicStageStats aggregateBasicStageStats(Iterable completedDrivers += stageStats.getCompletedDrivers(); cumulativeUserMemory += stageStats.getCumulativeUserMemory(); - cumulativeSystemMemory += stageStats.getCumulativeSystemMemory(); userMemoryReservation += stageStats.getUserMemoryReservation().toBytes(); totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes(); @@ -325,7 +314,6 @@ public static BasicStageStats aggregateBasicStageStats(Iterable rawInputPositions, cumulativeUserMemory, - cumulativeSystemMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), diff --git a/core/trino-main/src/main/java/io/trino/execution/MemoryTrackingRemoteTaskFactory.java b/core/trino-main/src/main/java/io/trino/execution/MemoryTrackingRemoteTaskFactory.java index 71b06e66fa5c..bfe929b234e8 100644 --- a/core/trino-main/src/main/java/io/trino/execution/MemoryTrackingRemoteTaskFactory.java +++ b/core/trino-main/src/main/java/io/trino/execution/MemoryTrackingRemoteTaskFactory.java @@ -71,7 +71,6 @@ private static final class UpdatePeakMemory { private final QueryStateMachine stateMachine; private long previousUserMemory; - private long previousSystemMemory; private long previousRevocableMemory; public UpdatePeakMemory(QueryStateMachine stateMachine) @@ -83,14 +82,12 @@ public UpdatePeakMemory(QueryStateMachine stateMachine) public synchronized void stateChanged(TaskStatus newStatus) { long currentUserMemory = newStatus.getMemoryReservation().toBytes(); - long currentSystemMemory = newStatus.getSystemMemoryReservation().toBytes(); long currentRevocableMemory = newStatus.getRevocableMemoryReservation().toBytes(); - long currentTotalMemory = currentUserMemory + currentSystemMemory + currentRevocableMemory; + long currentTotalMemory = currentUserMemory + currentRevocableMemory; long deltaUserMemoryInBytes = currentUserMemory - previousUserMemory; long deltaRevocableMemoryInBytes = currentRevocableMemory - previousRevocableMemory; - long deltaTotalMemoryInBytes = currentTotalMemory - (previousUserMemory + previousSystemMemory + previousRevocableMemory); + long deltaTotalMemoryInBytes = currentTotalMemory - (previousUserMemory + previousRevocableMemory); previousUserMemory = currentUserMemory; - previousSystemMemory = currentSystemMemory; previousRevocableMemory = currentRevocableMemory; stateMachine.updateMemoryUsage(deltaUserMemoryInBytes, deltaRevocableMemoryInBytes, deltaTotalMemoryInBytes, currentUserMemory, currentRevocableMemory, currentTotalMemory); } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java b/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java index 525b6ea67e0a..6352ed25f6ad 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java @@ -119,7 +119,6 @@ public class QueryStateMachine private final AtomicLong currentRevocableMemory = new AtomicLong(); private final AtomicLong peakRevocableMemory = new AtomicLong(); - private final AtomicLong peakNonRevocableMemory = new AtomicLong(); // peak of the user + system + revocable memory reservation private final AtomicLong currentTotalMemory = new AtomicLong(); @@ -289,11 +288,6 @@ public long getPeakRevocableMemoryInBytes() return peakRevocableMemory.get(); } - public long getPeakNonRevocableMemoryInBytes() - { - return peakNonRevocableMemory.get(); - } - public long getPeakTotalMemoryInBytes() { return peakTotalMemory.get(); @@ -332,7 +326,6 @@ public void updateMemoryUsage( currentTotalMemory.addAndGet(deltaTotalMemoryInBytes); peakUserMemory.updateAndGet(currentPeakValue -> Math.max(currentUserMemory.get(), currentPeakValue)); peakRevocableMemory.updateAndGet(currentPeakValue -> Math.max(currentRevocableMemory.get(), currentPeakValue)); - peakNonRevocableMemory.updateAndGet(currentPeakValue -> Math.max(currentTotalMemory.get() - currentRevocableMemory.get(), currentPeakValue)); peakTotalMemory.updateAndGet(currentPeakValue -> Math.max(currentTotalMemory.get(), currentPeakValue)); peakTaskUserMemory.accumulateAndGet(taskUserMemoryInBytes, Math::max); peakTaskRevocableMemory.accumulateAndGet(taskRevocableMemoryInBytes, Math::max); @@ -373,7 +366,6 @@ public BasicQueryInfo getBasicQueryInfo(Optional rootStage) stageStats.getPhysicalInputDataSize(), stageStats.getCumulativeUserMemory(), - stageStats.getCumulativeSystemMemory(), stageStats.getUserMemoryReservation(), stageStats.getTotalMemoryReservation(), succinctBytes(getPeakUserMemoryInBytes()), @@ -472,7 +464,6 @@ private QueryStats getQueryStats(Optional rootStage) int completedDrivers = 0; long cumulativeUserMemory = 0; - long cumulativeSystemMemory = 0; long userMemoryReservation = 0; long revocableMemoryReservation = 0; long totalMemoryReservation = 0; @@ -519,7 +510,6 @@ private QueryStats getQueryStats(Optional rootStage) completedDrivers += stageStats.getCompletedDrivers(); cumulativeUserMemory += stageStats.getCumulativeUserMemory(); - cumulativeSystemMemory += stageStats.getCumulativeSystemMemory(); userMemoryReservation += stageStats.getUserMemoryReservation().toBytes(); revocableMemoryReservation += stageStats.getRevocableMemoryReservation().toBytes(); totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes(); @@ -589,13 +579,11 @@ private QueryStats getQueryStats(Optional rootStage) completedDrivers, cumulativeUserMemory, - cumulativeSystemMemory, succinctBytes(userMemoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(totalMemoryReservation), succinctBytes(getPeakUserMemoryInBytes()), succinctBytes(getPeakRevocableMemoryInBytes()), - succinctBytes(getPeakNonRevocableMemoryInBytes()), succinctBytes(getPeakTotalMemoryInBytes()), succinctBytes(getPeakTaskUserMemory()), succinctBytes(getPeakTaskRevocableMemory()), @@ -1153,13 +1141,11 @@ private static QueryStats pruneQueryStats(QueryStats queryStats) queryStats.getBlockedDrivers(), queryStats.getCompletedDrivers(), queryStats.getCumulativeUserMemory(), - queryStats.getCumulativeSystemMemory(), queryStats.getUserMemoryReservation(), queryStats.getRevocableMemoryReservation(), queryStats.getTotalMemoryReservation(), queryStats.getPeakUserMemoryReservation(), queryStats.getPeakRevocableMemoryReservation(), - queryStats.getPeakNonRevocableMemoryReservation(), queryStats.getPeakTotalMemoryReservation(), queryStats.getPeakTaskUserMemory(), queryStats.getPeakTaskRevocableMemory(), diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryStats.java b/core/trino-main/src/main/java/io/trino/execution/QueryStats.java index 3a115a7f0a56..3631fab377aa 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryStats.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryStats.java @@ -65,13 +65,11 @@ public class QueryStats private final int completedDrivers; private final double cumulativeUserMemory; - private final double cumulativeSystemMemory; private final DataSize userMemoryReservation; private final DataSize revocableMemoryReservation; private final DataSize totalMemoryReservation; private final DataSize peakUserMemoryReservation; private final DataSize peakRevocableMemoryReservation; - private final DataSize peakNonRevocableMemoryReservation; private final DataSize peakTotalMemoryReservation; private final DataSize peakTaskUserMemory; private final DataSize peakTaskRevocableMemory; @@ -135,13 +133,11 @@ public QueryStats( @JsonProperty("completedDrivers") int completedDrivers, @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, - @JsonProperty("cumulativeSystemMemory") double cumulativeSystemMemory, @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation, @JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation, @JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation, @JsonProperty("peakRevocableMemoryReservation") DataSize peakRevocableMemoryReservation, - @JsonProperty("peakNonRevocableMemoryReservation") DataSize peakNonRevocableMemoryReservation, @JsonProperty("peakTotalMemoryReservation") DataSize peakTotalMemoryReservation, @JsonProperty("peakTaskUserMemory") DataSize peakTaskUserMemory, @JsonProperty("peakTaskRevocableMemory") DataSize peakTaskRevocableMemory, @@ -211,14 +207,11 @@ public QueryStats( this.completedDrivers = completedDrivers; checkArgument(cumulativeUserMemory >= 0, "cumulativeUserMemory is negative"); this.cumulativeUserMemory = cumulativeUserMemory; - checkArgument(cumulativeSystemMemory >= 0, "cumulativeSystemMemory is negative"); - this.cumulativeSystemMemory = cumulativeSystemMemory; this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null"); this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null"); this.totalMemoryReservation = requireNonNull(totalMemoryReservation, "totalMemoryReservation is null"); this.peakUserMemoryReservation = requireNonNull(peakUserMemoryReservation, "peakUserMemoryReservation is null"); this.peakRevocableMemoryReservation = requireNonNull(peakRevocableMemoryReservation, "peakRevocableMemoryReservation is null"); - this.peakNonRevocableMemoryReservation = requireNonNull(peakNonRevocableMemoryReservation, "peakNonRevocableMemoryReservation is null"); this.peakTotalMemoryReservation = requireNonNull(peakTotalMemoryReservation, "peakTotalMemoryReservation is null"); this.peakTaskUserMemory = requireNonNull(peakTaskUserMemory, "peakTaskUserMemory is null"); this.peakTaskRevocableMemory = requireNonNull(peakTaskRevocableMemory, "peakTaskRevocableMemory is null"); @@ -387,12 +380,6 @@ public double getCumulativeUserMemory() return cumulativeUserMemory; } - @JsonProperty - public double getCumulativeSystemMemory() - { - return cumulativeSystemMemory; - } - @JsonProperty public DataSize getUserMemoryReservation() { @@ -423,12 +410,6 @@ public DataSize getPeakRevocableMemoryReservation() return peakRevocableMemoryReservation; } - @JsonProperty - public DataSize getPeakNonRevocableMemoryReservation() - { - return peakNonRevocableMemoryReservation; - } - @JsonProperty public DataSize getPeakTotalMemoryReservation() { diff --git a/core/trino-main/src/main/java/io/trino/execution/SqlStage.java b/core/trino-main/src/main/java/io/trino/execution/SqlStage.java index f03d18cb3fc2..82ab33861f81 100644 --- a/core/trino-main/src/main/java/io/trino/execution/SqlStage.java +++ b/core/trino-main/src/main/java/io/trino/execution/SqlStage.java @@ -292,20 +292,17 @@ private class MemoryUsageListener implements StateChangeListener { private long previousUserMemory; - private long previousSystemMemory; private long previousRevocableMemory; @Override public synchronized void stateChanged(TaskStatus taskStatus) { long currentUserMemory = taskStatus.getMemoryReservation().toBytes(); - long currentSystemMemory = taskStatus.getSystemMemoryReservation().toBytes(); long currentRevocableMemory = taskStatus.getRevocableMemoryReservation().toBytes(); long deltaUserMemoryInBytes = currentUserMemory - previousUserMemory; long deltaRevocableMemoryInBytes = currentRevocableMemory - previousRevocableMemory; - long deltaTotalMemoryInBytes = (currentUserMemory + currentSystemMemory + currentRevocableMemory) - (previousUserMemory + previousSystemMemory + previousRevocableMemory); + long deltaTotalMemoryInBytes = (currentUserMemory + currentRevocableMemory) - (previousUserMemory + previousRevocableMemory); previousUserMemory = currentUserMemory; - previousSystemMemory = currentSystemMemory; previousRevocableMemory = currentRevocableMemory; stateMachine.updateMemoryUsage(deltaUserMemoryInBytes, deltaRevocableMemoryInBytes, deltaTotalMemoryInBytes); } diff --git a/core/trino-main/src/main/java/io/trino/execution/SqlTask.java b/core/trino-main/src/main/java/io/trino/execution/SqlTask.java index 28f49afe1272..1c5176626063 100644 --- a/core/trino-main/src/main/java/io/trino/execution/SqlTask.java +++ b/core/trino-main/src/main/java/io/trino/execution/SqlTask.java @@ -140,7 +140,7 @@ private SqlTask( maxBroadcastBufferSize, // Pass a memory context supplier instead of a memory context to the output buffer, // because we haven't created the task context that holds the memory context yet. - () -> queryContext.getTaskContextByTaskId(taskId).localSystemMemoryContext(), + () -> queryContext.getTaskContextByTaskId(taskId).localMemoryContext(), () -> notifyStatusChanged(), exchangeManagerRegistry); taskStateMachine = new TaskStateMachine(taskId, taskNotificationExecutor); @@ -290,7 +290,6 @@ private TaskStatus createTaskStatus(TaskHolder taskHolder) long runningPartitionedSplitsWeight = 0L; DataSize physicalWrittenDataSize = DataSize.ofBytes(0); DataSize userMemoryReservation = DataSize.ofBytes(0); - DataSize systemMemoryReservation = DataSize.ofBytes(0); DataSize revocableMemoryReservation = DataSize.ofBytes(0); // TODO: add a mechanism to avoid sending the whole completedDriverGroups set over the wire for every task status reply Set completedDriverGroups = ImmutableSet.of(); @@ -306,7 +305,6 @@ private TaskStatus createTaskStatus(TaskHolder taskHolder) runningPartitionedSplitsWeight = taskStats.getRunningPartitionedSplitsWeight(); physicalWrittenDataSize = taskStats.getPhysicalWrittenDataSize(); userMemoryReservation = taskStats.getUserMemoryReservation(); - systemMemoryReservation = taskStats.getSystemMemoryReservation(); revocableMemoryReservation = taskStats.getRevocableMemoryReservation(); fullGcCount = taskStats.getFullGcCount(); fullGcTime = taskStats.getFullGcTime(); @@ -324,7 +322,6 @@ else if (taskHolder.getTaskExecution() != null) { } physicalWrittenDataSize = succinctBytes(physicalWrittenBytes); userMemoryReservation = taskContext.getMemoryReservation(); - systemMemoryReservation = taskContext.getSystemMemoryReservation(); revocableMemoryReservation = taskContext.getRevocableMemoryReservation(); completedDriverGroups = taskContext.getCompletedDriverGroups(); fullGcCount = taskContext.getFullGcCount(); @@ -345,7 +342,6 @@ else if (taskHolder.getTaskExecution() != null) { isOutputBufferOverutilized(), physicalWrittenDataSize, userMemoryReservation, - systemMemoryReservation, revocableMemoryReservation, fullGcCount, fullGcTime, diff --git a/core/trino-main/src/main/java/io/trino/execution/SqlTaskManager.java b/core/trino-main/src/main/java/io/trino/execution/SqlTaskManager.java index 564db52e75b4..38352a26e149 100644 --- a/core/trino-main/src/main/java/io/trino/execution/SqlTaskManager.java +++ b/core/trino-main/src/main/java/io/trino/execution/SqlTaskManager.java @@ -75,7 +75,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.concurrent.Threads.threadsNamed; import static io.trino.SystemSessionProperties.getQueryMaxMemoryPerNode; -import static io.trino.SystemSessionProperties.getQueryMaxTotalMemoryPerNode; import static io.trino.SystemSessionProperties.getQueryMaxTotalMemoryPerTask; import static io.trino.SystemSessionProperties.resourceOvercommit; import static io.trino.execution.SqlTask.createSqlTask; @@ -113,7 +112,6 @@ public class SqlTaskManager private final SqlTaskIoStats finishedTaskStats = new SqlTaskIoStats(); private final long queryMaxMemoryPerNode; - private final long queryMaxTotalMemoryPerNode; private final Optional queryMaxMemoryPerTask; @GuardedBy("this") @@ -159,15 +157,13 @@ public SqlTaskManager( this.localMemoryManager = requireNonNull(localMemoryManager, "localMemoryManager is null"); DataSize maxQueryMemoryPerNode = nodeMemoryConfig.getMaxQueryMemoryPerNode(); - DataSize maxQueryTotalMemoryPerNode = nodeMemoryConfig.getMaxQueryTotalMemoryPerNode(); - queryMaxMemoryPerTask = nodeMemoryConfig.getMaxQueryTotalMemoryPerTask(); + queryMaxMemoryPerTask = nodeMemoryConfig.getMaxQueryMemoryPerTask(); DataSize maxQuerySpillPerNode = nodeSpillConfig.getQueryMaxSpillPerNode(); queryMaxMemoryPerNode = maxQueryMemoryPerNode.toBytes(); - queryMaxTotalMemoryPerNode = maxQueryTotalMemoryPerNode.toBytes(); queryContexts = buildNonEvictableCache(CacheBuilder.newBuilder().weakValues(), CacheLoader.from( - queryId -> createQueryContext(queryId, localMemoryManager, localSpillManager, gcMonitor, maxQueryMemoryPerNode, maxQueryTotalMemoryPerNode, queryMaxMemoryPerTask, maxQuerySpillPerNode))); + queryId -> createQueryContext(queryId, localMemoryManager, localSpillManager, gcMonitor, maxQueryMemoryPerNode, queryMaxMemoryPerTask, maxQuerySpillPerNode))); tasks = buildNonEvictableCache(CacheBuilder.newBuilder(), CacheLoader.from( taskId -> createSqlTask( @@ -190,14 +186,12 @@ private QueryContext createQueryContext( LocalSpillManager localSpillManager, GcMonitor gcMonitor, DataSize maxQueryUserMemoryPerNode, - DataSize maxQueryTotalMemoryPerNode, Optional maxQueryMemoryPerTask, DataSize maxQuerySpillPerNode) { return new QueryContext( queryId, maxQueryUserMemoryPerNode, - maxQueryTotalMemoryPerNode, maxQueryMemoryPerTask, localMemoryManager.getGeneralPool(), gcMonitor, @@ -414,7 +408,6 @@ private TaskInfo doUpdateTask( QueryContext queryContext = sqlTask.getQueryContext(); if (!queryContext.isMemoryLimitsInitialized()) { long sessionQueryMaxMemoryPerNode = getQueryMaxMemoryPerNode(session).toBytes(); - long sessionQueryTotalMaxMemoryPerNode = getQueryMaxTotalMemoryPerNode(session).toBytes(); Optional effectiveQueryMaxMemoryPerTask = getQueryMaxTotalMemoryPerTask(session); if (queryMaxMemoryPerTask.isPresent() && @@ -426,7 +419,6 @@ private TaskInfo doUpdateTask( queryContext.initializeMemoryLimits( resourceOvercommit(session), min(sessionQueryMaxMemoryPerNode, queryMaxMemoryPerNode), - min(sessionQueryTotalMaxMemoryPerNode, queryMaxTotalMemoryPerNode), effectiveQueryMaxMemoryPerTask); } diff --git a/core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java b/core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java index ff3885dc369c..950b0fd8708d 100644 --- a/core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java +++ b/core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java @@ -242,7 +242,6 @@ public BasicStageStats getBasicStageStats(Supplier> taskInfos int completedDrivers = 0; long cumulativeUserMemory = 0; - long cumulativeSystemMemory = 0; long userMemoryReservation = 0; long totalMemoryReservation = 0; @@ -272,13 +271,11 @@ public BasicStageStats getBasicStageStats(Supplier> taskInfos completedDrivers += taskStats.getCompletedDrivers(); cumulativeUserMemory += taskStats.getCumulativeUserMemory(); - cumulativeSystemMemory += taskStats.getCumulativeSystemMemory(); long taskUserMemory = taskStats.getUserMemoryReservation().toBytes(); - long taskSystemMemory = taskStats.getSystemMemoryReservation().toBytes(); long taskRevocableMemory = taskStats.getRevocableMemoryReservation().toBytes(); userMemoryReservation += taskUserMemory; - totalMemoryReservation += taskUserMemory + taskSystemMemory + taskRevocableMemory; + totalMemoryReservation += taskUserMemory + taskRevocableMemory; totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS); totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS); @@ -324,7 +321,6 @@ public BasicStageStats getBasicStageStats(Supplier> taskInfos rawInputPositions, cumulativeUserMemory, - cumulativeSystemMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), @@ -364,7 +360,6 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier) int completedDrivers = 0; long cumulativeUserMemory = 0; - long cumulativeSystemMemory = 0; long userMemoryReservation = 0; long revocableMemoryReservation = 0; long totalMemoryReservation = 0; @@ -426,14 +421,12 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier) completedDrivers += taskStats.getCompletedDrivers(); cumulativeUserMemory += taskStats.getCumulativeUserMemory(); - cumulativeSystemMemory += taskStats.getCumulativeSystemMemory(); long taskUserMemory = taskStats.getUserMemoryReservation().toBytes(); - long taskSystemMemory = taskStats.getSystemMemoryReservation().toBytes(); long taskRevocableMemory = taskStats.getRevocableMemoryReservation().toBytes(); userMemoryReservation += taskUserMemory; revocableMemoryReservation += taskRevocableMemory; - totalMemoryReservation += taskUserMemory + taskSystemMemory + taskRevocableMemory; + totalMemoryReservation += taskUserMemory + taskRevocableMemory; totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS); totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS); @@ -494,7 +487,6 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier) completedDrivers, cumulativeUserMemory, - cumulativeSystemMemory, succinctBytes(userMemoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(totalMemoryReservation), diff --git a/core/trino-main/src/main/java/io/trino/execution/StageStats.java b/core/trino-main/src/main/java/io/trino/execution/StageStats.java index d7d1c6645913..efb457f701df 100644 --- a/core/trino-main/src/main/java/io/trino/execution/StageStats.java +++ b/core/trino-main/src/main/java/io/trino/execution/StageStats.java @@ -55,7 +55,6 @@ public class StageStats private final int completedDrivers; private final double cumulativeUserMemory; - private final double cumulativeSystemMemory; private final DataSize userMemoryReservation; private final DataSize revocableMemoryReservation; private final DataSize totalMemoryReservation; @@ -109,7 +108,6 @@ public StageStats( @JsonProperty("completedDrivers") int completedDrivers, @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, - @JsonProperty("cumulativeSystemMemory") double cumulativeSystemMemory, @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation, @JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation, @@ -169,8 +167,6 @@ public StageStats( this.completedDrivers = completedDrivers; checkArgument(cumulativeUserMemory >= 0, "cumulativeUserMemory is negative"); this.cumulativeUserMemory = cumulativeUserMemory; - checkArgument(cumulativeSystemMemory >= 0, "cumulativeSystemMemory is negative"); - this.cumulativeSystemMemory = cumulativeSystemMemory; this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null"); this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null"); this.totalMemoryReservation = requireNonNull(totalMemoryReservation, "totalMemoryReservation is null"); @@ -284,12 +280,6 @@ public double getCumulativeUserMemory() return cumulativeUserMemory; } - @JsonProperty - public double getCumulativeSystemMemory() - { - return cumulativeSystemMemory; - } - @JsonProperty public DataSize getUserMemoryReservation() { @@ -463,7 +453,6 @@ public BasicStageStats toBasicStageStats(StageState stageState) rawInputDataSize, rawInputPositions, (long) cumulativeUserMemory, - (long) cumulativeSystemMemory, userMemoryReservation, totalMemoryReservation, totalCpuTime, diff --git a/core/trino-main/src/main/java/io/trino/execution/TaskStatus.java b/core/trino-main/src/main/java/io/trino/execution/TaskStatus.java index dfcd4136aac9..4a421a1456e5 100644 --- a/core/trino-main/src/main/java/io/trino/execution/TaskStatus.java +++ b/core/trino-main/src/main/java/io/trino/execution/TaskStatus.java @@ -61,7 +61,6 @@ public class TaskStatus private final boolean outputBufferOverutilized; private final DataSize physicalWrittenDataSize; private final DataSize memoryReservation; - private final DataSize systemMemoryReservation; private final DataSize revocableMemoryReservation; private final long fullGcCount; @@ -86,7 +85,6 @@ public TaskStatus( @JsonProperty("outputBufferOverutilized") boolean outputBufferOverutilized, @JsonProperty("physicalWrittenDataSize") DataSize physicalWrittenDataSize, @JsonProperty("memoryReservation") DataSize memoryReservation, - @JsonProperty("systemMemoryReservation") DataSize systemMemoryReservation, @JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation, @JsonProperty("fullGcCount") long fullGcCount, @JsonProperty("fullGcTime") Duration fullGcTime, @@ -119,7 +117,6 @@ public TaskStatus( this.physicalWrittenDataSize = requireNonNull(physicalWrittenDataSize, "physicalWrittenDataSize is null"); this.memoryReservation = requireNonNull(memoryReservation, "memoryReservation is null"); - this.systemMemoryReservation = requireNonNull(systemMemoryReservation, "systemMemoryReservation is null"); this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null"); this.failures = ImmutableList.copyOf(requireNonNull(failures, "failures is null")); @@ -208,12 +205,6 @@ public DataSize getMemoryReservation() return memoryReservation; } - @JsonProperty - public DataSize getSystemMemoryReservation() - { - return systemMemoryReservation; - } - @JsonProperty public DataSize getRevocableMemoryReservation() { @@ -276,7 +267,6 @@ public static TaskStatus initialTaskStatus(TaskId taskId, URI location, String n DataSize.ofBytes(0), DataSize.ofBytes(0), DataSize.ofBytes(0), - DataSize.ofBytes(0), 0, new Duration(0, MILLISECONDS), INITIAL_DYNAMIC_FILTERS_VERSION, @@ -300,7 +290,6 @@ public static TaskStatus failWith(TaskStatus taskStatus, TaskState state, List state, DataSize maxBufferSize, - Supplier systemMemoryContextSupplier, + Supplier memoryContextSupplier, Executor notificationExecutor) { this.taskInstanceId = requireNonNull(taskInstanceId, "taskInstanceId is null"); @@ -96,7 +96,7 @@ public ArbitraryOutputBuffer( checkArgument(maxBufferSize.toBytes() > 0, "maxBufferSize must be at least 1"); this.memoryManager = new OutputBufferMemoryManager( maxBufferSize.toBytes(), - requireNonNull(systemMemoryContextSupplier, "systemMemoryContextSupplier is null"), + requireNonNull(memoryContextSupplier, "memoryContextSupplier is null"), requireNonNull(notificationExecutor, "notificationExecutor is null")); this.onPagesReleased = PagesReleasedListener.forOutputBufferMemoryManager(memoryManager); this.masterBuffer = new MasterBuffer(onPagesReleased); diff --git a/core/trino-main/src/main/java/io/trino/execution/buffer/BroadcastOutputBuffer.java b/core/trino-main/src/main/java/io/trino/execution/buffer/BroadcastOutputBuffer.java index 1c0f9e5329e0..7dd749199dff 100644 --- a/core/trino-main/src/main/java/io/trino/execution/buffer/BroadcastOutputBuffer.java +++ b/core/trino-main/src/main/java/io/trino/execution/buffer/BroadcastOutputBuffer.java @@ -81,7 +81,7 @@ public BroadcastOutputBuffer( String taskInstanceId, StateMachine state, DataSize maxBufferSize, - Supplier systemMemoryContextSupplier, + Supplier memoryContextSupplier, Executor notificationExecutor, Runnable notifyStatusChanged) { @@ -89,7 +89,7 @@ public BroadcastOutputBuffer( this.state = requireNonNull(state, "state is null"); this.memoryManager = new OutputBufferMemoryManager( requireNonNull(maxBufferSize, "maxBufferSize is null").toBytes(), - requireNonNull(systemMemoryContextSupplier, "systemMemoryContextSupplier is null"), + requireNonNull(memoryContextSupplier, "memoryContextSupplier is null"), requireNonNull(notificationExecutor, "notificationExecutor is null")); this.onPagesReleased = (releasedPageCount, releasedMemorySizeInBytes) -> { checkState(totalBufferedPages.addAndGet(-releasedPageCount) >= 0); diff --git a/core/trino-main/src/main/java/io/trino/execution/buffer/LazyOutputBuffer.java b/core/trino-main/src/main/java/io/trino/execution/buffer/LazyOutputBuffer.java index 8be374b698ae..762d6f7af08e 100644 --- a/core/trino-main/src/main/java/io/trino/execution/buffer/LazyOutputBuffer.java +++ b/core/trino-main/src/main/java/io/trino/execution/buffer/LazyOutputBuffer.java @@ -56,7 +56,7 @@ public class LazyOutputBuffer private final String taskInstanceId; private final DataSize maxBufferSize; private final DataSize maxBroadcastBufferSize; - private final Supplier systemMemoryContextSupplier; + private final Supplier memoryContextSupplier; private final Executor executor; private final Runnable notifyStatusChanged; private final ExchangeManagerRegistry exchangeManagerRegistry; @@ -78,7 +78,7 @@ public LazyOutputBuffer( Executor executor, DataSize maxBufferSize, DataSize maxBroadcastBufferSize, - Supplier systemMemoryContextSupplier, + Supplier memoryContextSupplier, Runnable notifyStatusChanged, ExchangeManagerRegistry exchangeManagerRegistry) { @@ -88,7 +88,7 @@ public LazyOutputBuffer( this.maxBufferSize = requireNonNull(maxBufferSize, "maxBufferSize is null"); this.maxBroadcastBufferSize = requireNonNull(maxBroadcastBufferSize, "maxBroadcastBufferSize is null"); checkArgument(maxBufferSize.toBytes() > 0, "maxBufferSize must be at least 1"); - this.systemMemoryContextSupplier = requireNonNull(systemMemoryContextSupplier, "systemMemoryContextSupplier is null"); + this.memoryContextSupplier = requireNonNull(memoryContextSupplier, "memoryContextSupplier is null"); this.notifyStatusChanged = requireNonNull(notifyStatusChanged, "notifyStatusChanged is null"); this.exchangeManagerRegistry = requireNonNull(exchangeManagerRegistry, "exchangeManagerRegistry is null"); } @@ -167,20 +167,20 @@ public void setOutputBuffers(OutputBuffers newOutputBuffers) } switch (newOutputBuffers.getType()) { case PARTITIONED: - outputBuffer = new PartitionedOutputBuffer(taskInstanceId, state, newOutputBuffers, maxBufferSize, systemMemoryContextSupplier, executor); + outputBuffer = new PartitionedOutputBuffer(taskInstanceId, state, newOutputBuffers, maxBufferSize, memoryContextSupplier, executor); break; case BROADCAST: - outputBuffer = new BroadcastOutputBuffer(taskInstanceId, state, maxBroadcastBufferSize, systemMemoryContextSupplier, executor, notifyStatusChanged); + outputBuffer = new BroadcastOutputBuffer(taskInstanceId, state, maxBroadcastBufferSize, memoryContextSupplier, executor, notifyStatusChanged); break; case ARBITRARY: - outputBuffer = new ArbitraryOutputBuffer(taskInstanceId, state, maxBufferSize, systemMemoryContextSupplier, executor); + outputBuffer = new ArbitraryOutputBuffer(taskInstanceId, state, maxBufferSize, memoryContextSupplier, executor); break; case SPOOL: ExchangeSinkInstanceHandle exchangeSinkInstanceHandle = newOutputBuffers.getExchangeSinkInstanceHandle() .orElseThrow(() -> new IllegalArgumentException("exchange sink handle is expected to be present for buffer type EXTERNAL")); ExchangeManager exchangeManager = exchangeManagerRegistry.getExchangeManager(); ExchangeSink exchangeSink = exchangeManager.createSink(exchangeSinkInstanceHandle); - outputBuffer = new SpoolingExchangeOutputBuffer(state, newOutputBuffers, exchangeSink, systemMemoryContextSupplier); + outputBuffer = new SpoolingExchangeOutputBuffer(state, newOutputBuffers, exchangeSink, memoryContextSupplier); break; default: throw new IllegalArgumentException("Unexpected output buffer type: " + newOutputBuffers.getType()); diff --git a/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferMemoryManager.java b/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferMemoryManager.java index 20b6541851a2..b805b86780e2 100644 --- a/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferMemoryManager.java +++ b/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferMemoryManager.java @@ -56,24 +56,24 @@ class OutputBufferMemoryManager private final AtomicBoolean blockOnFull = new AtomicBoolean(true); - private final Supplier systemMemoryContextSupplier; + private final Supplier memoryContextSupplier; private final Executor notificationExecutor; - public OutputBufferMemoryManager(long maxBufferedBytes, Supplier systemMemoryContextSupplier, Executor notificationExecutor) + public OutputBufferMemoryManager(long maxBufferedBytes, Supplier memoryContextSupplier, Executor notificationExecutor) { - requireNonNull(systemMemoryContextSupplier, "systemMemoryContextSupplier is null"); + requireNonNull(memoryContextSupplier, "memoryContextSupplier is null"); checkArgument(maxBufferedBytes > 0, "maxBufferedBytes must be > 0"); this.maxBufferedBytes = maxBufferedBytes; - this.systemMemoryContextSupplier = Suppliers.memoize(systemMemoryContextSupplier::get); + this.memoryContextSupplier = Suppliers.memoize(memoryContextSupplier::get); this.notificationExecutor = requireNonNull(notificationExecutor, "notificationExecutor is null"); } public void updateMemoryUsage(long bytesAdded) { - // If the systemMemoryContext doesn't exist, the task is probably already - // aborted, so we can just return (see the comment in getSystemMemoryContextOrNull()). - LocalMemoryContext systemMemoryContext = getSystemMemoryContextOrNull(); - if (systemMemoryContext == null) { + // If the memoryContext doesn't exist, the task is probably already + // aborted, so we can just return (see the comment in getMemoryContextOrNull()). + LocalMemoryContext memoryContext = getMemoryContextOrNull(); + if (memoryContext == null) { return; } @@ -93,7 +93,7 @@ public void updateMemoryUsage(long bytesAdded) checkArgument(result >= 0, "bufferedBytes (%s) plus delta (%s) would be negative", bytes, bytesAdded); return result; }); - ListenableFuture blockedOnMemory = systemMemoryContext.setBytes(currentBufferedBytes); + ListenableFuture blockedOnMemory = memoryContext.setBytes(currentBufferedBytes); if (!blockedOnMemory.isDone()) { if (this.blockedOnMemory != blockedOnMemory) { this.blockedOnMemory = blockedOnMemory; @@ -194,7 +194,7 @@ public long getPeakMemoryUsage() public synchronized void close() { updateMemoryUsage(-bufferedBytes.get()); - LocalMemoryContext memoryContext = getSystemMemoryContextOrNull(); + LocalMemoryContext memoryContext = getMemoryContextOrNull(); if (memoryContext != null) { memoryContext.close(); } @@ -209,10 +209,10 @@ private void notifyListener(@Nullable SettableFuture future) } @Nullable - private LocalMemoryContext getSystemMemoryContextOrNull() + private LocalMemoryContext getMemoryContextOrNull() { try { - return systemMemoryContextSupplier.get(); + return memoryContextSupplier.get(); } catch (RuntimeException ignored) { // This is possible with races, e.g., a task is created and then immediately aborted, diff --git a/core/trino-main/src/main/java/io/trino/execution/buffer/PartitionedOutputBuffer.java b/core/trino-main/src/main/java/io/trino/execution/buffer/PartitionedOutputBuffer.java index e307fb8582ac..b0771437c296 100644 --- a/core/trino-main/src/main/java/io/trino/execution/buffer/PartitionedOutputBuffer.java +++ b/core/trino-main/src/main/java/io/trino/execution/buffer/PartitionedOutputBuffer.java @@ -60,7 +60,7 @@ public PartitionedOutputBuffer( StateMachine state, OutputBuffers outputBuffers, DataSize maxBufferSize, - Supplier systemMemoryContextSupplier, + Supplier memoryContextSupplier, Executor notificationExecutor) { this.state = requireNonNull(state, "state is null"); @@ -71,7 +71,7 @@ public PartitionedOutputBuffer( this.outputBuffers = outputBuffers; this.memoryManager = new OutputBufferMemoryManager( requireNonNull(maxBufferSize, "maxBufferSize is null").toBytes(), - requireNonNull(systemMemoryContextSupplier, "systemMemoryContextSupplier is null"), + requireNonNull(memoryContextSupplier, "memoryContextSupplier is null"), requireNonNull(notificationExecutor, "notificationExecutor is null")); this.onPagesReleased = PagesReleasedListener.forOutputBufferMemoryManager(memoryManager); diff --git a/core/trino-main/src/main/java/io/trino/memory/LocalMemoryManager.java b/core/trino-main/src/main/java/io/trino/memory/LocalMemoryManager.java index 584243bba255..782830202155 100644 --- a/core/trino-main/src/main/java/io/trino/memory/LocalMemoryManager.java +++ b/core/trino-main/src/main/java/io/trino/memory/LocalMemoryManager.java @@ -28,10 +28,7 @@ import java.util.Map; import java.util.Optional; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Verify.verify; -import static io.trino.memory.NodeMemoryConfig.QUERY_MAX_MEMORY_PER_NODE_CONFIG; -import static io.trino.memory.NodeMemoryConfig.QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -61,16 +58,11 @@ private void configureMemoryPools(NodeMemoryConfig config, long availableMemory) { validateHeapHeadroom(config, availableMemory); maxMemory = DataSize.ofBytes(availableMemory - config.getHeapHeadroom().toBytes()); - checkArgument( - config.getMaxQueryMemoryPerNode().toBytes() <= config.getMaxQueryTotalMemoryPerNode().toBytes(), - "Max query memory per node (%s) cannot be greater than the max query total memory per node (%s).", - QUERY_MAX_MEMORY_PER_NODE_CONFIG, - QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG); ImmutableMap.Builder builder = ImmutableMap.builder(); long generalPoolSize = maxMemory.toBytes(); if (!config.isReservedPoolDisabled()) { - builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryTotalMemoryPerNode())); - generalPoolSize -= config.getMaxQueryTotalMemoryPerNode().toBytes(); + builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryMemoryPerNode())); + generalPoolSize -= config.getMaxQueryMemoryPerNode().toBytes(); } verify(generalPoolSize > 0, "general memory pool size is 0"); builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, DataSize.ofBytes(generalPoolSize))); @@ -79,13 +71,13 @@ private void configureMemoryPools(NodeMemoryConfig config, long availableMemory) private void validateHeapHeadroom(NodeMemoryConfig config, long availableMemory) { - long maxQueryTotalMemoryPerNode = config.getMaxQueryTotalMemoryPerNode().toBytes(); + long maxQueryTotalMemoryPerNode = config.getMaxQueryMemoryPerNode().toBytes(); long heapHeadroom = config.getHeapHeadroom().toBytes(); // (availableMemory - maxQueryTotalMemoryPerNode) bytes will be available for the general pool and the // headroom/untracked allocations, so the heapHeadroom cannot be larger than that space. if (heapHeadroom < 0 || heapHeadroom + maxQueryTotalMemoryPerNode > availableMemory) { throw new IllegalArgumentException( - format("Invalid memory configuration. The sum of max total query memory per node (%s) and heap headroom (%s) cannot be larger than the available heap memory (%s)", + format("Invalid memory configuration. The sum of max query memory per node (%s) and heap headroom (%s) cannot be larger than the available heap memory (%s)", maxQueryTotalMemoryPerNode, heapHeadroom, availableMemory)); diff --git a/core/trino-main/src/main/java/io/trino/memory/NodeMemoryConfig.java b/core/trino-main/src/main/java/io/trino/memory/NodeMemoryConfig.java index 95b28245873c..d58054de0660 100644 --- a/core/trino-main/src/main/java/io/trino/memory/NodeMemoryConfig.java +++ b/core/trino-main/src/main/java/io/trino/memory/NodeMemoryConfig.java @@ -24,22 +24,22 @@ import java.util.Optional; // This is separate from MemoryManagerConfig because it's difficult to test the default value of maxQueryMemoryPerNode -@DefunctConfig("deprecated.legacy-system-pool-enabled") +@DefunctConfig({ + "deprecated.legacy-system-pool-enabled", + "query.max-total-memory-per-node", +}) public class NodeMemoryConfig { public static final long AVAILABLE_HEAP_MEMORY = Runtime.getRuntime().maxMemory(); public static final String QUERY_MAX_MEMORY_PER_NODE_CONFIG = "query.max-memory-per-node"; - public static final String QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG = "query.max-total-memory-per-node"; - public static final String QUERY_MAX_TOTAL_MEMORY_PER_TASK_CONFIG = "query.max-total-memory-per-task"; + public static final String QUERY_MAX_MEMORY_PER_TASK_CONFIG = "query.max-memory-per-task"; private boolean isReservedPoolDisabled = true; - private DataSize maxQueryMemoryPerNode = DataSize.ofBytes(Math.round(AVAILABLE_HEAP_MEMORY * 0.1)); + private DataSize maxQueryMemoryPerNode = DataSize.ofBytes(Math.round(AVAILABLE_HEAP_MEMORY * 0.3)); - private Optional maxQueryTotalMemoryPerTask = Optional.empty(); + private Optional maxQueryMemoryPerTask = Optional.empty(); - // This is a per-query limit for the user plus system allocations. - private DataSize maxQueryTotalMemoryPerNode = DataSize.ofBytes(Math.round(AVAILABLE_HEAP_MEMORY * 0.3)); private DataSize heapHeadroom = DataSize.ofBytes(Math.round(AVAILABLE_HEAP_MEMORY * 0.3)); @NotNull @@ -56,16 +56,17 @@ public NodeMemoryConfig setMaxQueryMemoryPerNode(DataSize maxQueryMemoryPerNode) } @NotNull - public Optional getMaxQueryTotalMemoryPerTask() + public Optional getMaxQueryMemoryPerTask() { - return maxQueryTotalMemoryPerTask; + return maxQueryMemoryPerTask; } - @Config(QUERY_MAX_TOTAL_MEMORY_PER_TASK_CONFIG) - @ConfigDescription("Sets total (user + system) memory limit enforced for a single task; there is no memory limit by default") - public NodeMemoryConfig setMaxQueryTotalMemoryPerTask(DataSize maxQueryTotalMemoryPerTask) + @Config(QUERY_MAX_MEMORY_PER_TASK_CONFIG) + @LegacyConfig("query.max-total-memory-per-task") + @ConfigDescription("Sets memory limit enforced for a single task; there is no memory limit by default") + public NodeMemoryConfig setMaxQueryMemoryPerTask(DataSize maxQueryMemoryPerTask) { - this.maxQueryTotalMemoryPerTask = Optional.ofNullable(maxQueryTotalMemoryPerTask); + this.maxQueryMemoryPerTask = Optional.ofNullable(maxQueryMemoryPerTask); return this; } @@ -90,19 +91,6 @@ public NodeMemoryConfig setReservedPoolDisabled(boolean reservedPoolDisabled) return this; } - @NotNull - public DataSize getMaxQueryTotalMemoryPerNode() - { - return maxQueryTotalMemoryPerNode; - } - - @Config(QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG) - public NodeMemoryConfig setMaxQueryTotalMemoryPerNode(DataSize maxQueryTotalMemoryPerNode) - { - this.maxQueryTotalMemoryPerNode = maxQueryTotalMemoryPerNode; - return this; - } - @NotNull public DataSize getHeapHeadroom() { diff --git a/core/trino-main/src/main/java/io/trino/memory/QueryContext.java b/core/trino-main/src/main/java/io/trino/memory/QueryContext.java index bd00e5efff80..8f5f468c4973 100644 --- a/core/trino-main/src/main/java/io/trino/memory/QueryContext.java +++ b/core/trino-main/src/main/java/io/trino/memory/QueryContext.java @@ -50,7 +50,6 @@ import static io.airlift.concurrent.MoreFutures.whenAnyComplete; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static io.airlift.units.DataSize.succinctBytes; -import static io.trino.ExceededMemoryLimitException.exceededLocalTotalMemoryLimit; import static io.trino.ExceededMemoryLimitException.exceededLocalUserMemoryLimit; import static io.trino.ExceededSpillLimitException.exceededPerQueryLocalLimit; import static io.trino.memory.context.AggregatedMemoryContext.newRootAggregatedMemoryContext; @@ -82,8 +81,6 @@ public class QueryContext @GuardedBy("this") private long maxUserMemory; @GuardedBy("this") - private long maxTotalMemory; - @GuardedBy("this") private Optional maxTaskMemory; private final MemoryTrackingContext queryMemoryContext; @@ -100,7 +97,6 @@ public class QueryContext public QueryContext( QueryId queryId, DataSize maxUserMemory, - DataSize maxTotalMemory, Optional maxTaskMemory, MemoryPool memoryPool, GcMonitor gcMonitor, @@ -108,10 +104,34 @@ public QueryContext( ScheduledExecutorService yieldExecutor, DataSize maxSpill, SpillSpaceTracker spillSpaceTracker) + { + this( + queryId, + maxUserMemory, + maxTaskMemory, + memoryPool, + GUARANTEED_MEMORY, + gcMonitor, + notificationExecutor, + yieldExecutor, + maxSpill, + spillSpaceTracker); + } + + public QueryContext( + QueryId queryId, + DataSize maxUserMemory, + Optional maxTaskMemory, + MemoryPool memoryPool, + long guaranteedMemory, + GcMonitor gcMonitor, + Executor notificationExecutor, + ScheduledExecutorService yieldExecutor, + DataSize maxSpill, + SpillSpaceTracker spillSpaceTracker) { this.queryId = requireNonNull(queryId, "queryId is null"); this.maxUserMemory = requireNonNull(maxUserMemory, "maxUserMemory is null").toBytes(); - this.maxTotalMemory = requireNonNull(maxTotalMemory, "maxTotalMemory is null").toBytes(); this.maxTaskMemory = requireNonNull(maxTaskMemory, "maxTaskMemory is null"); this.memoryPool = requireNonNull(memoryPool, "memoryPool is null"); this.gcMonitor = requireNonNull(gcMonitor, "gcMonitor is null"); @@ -120,9 +140,8 @@ public QueryContext( this.maxSpill = requireNonNull(maxSpill, "maxSpill is null").toBytes(); this.spillSpaceTracker = requireNonNull(spillSpaceTracker, "spillSpaceTracker is null"); this.queryMemoryContext = new MemoryTrackingContext( - newRootAggregatedMemoryContext(new QueryMemoryReservationHandler(this::updateUserMemory, this::tryUpdateUserMemory), GUARANTEED_MEMORY), - newRootAggregatedMemoryContext(new QueryMemoryReservationHandler(this::updateRevocableMemory, this::tryReserveMemoryNotSupported), 0L), - newRootAggregatedMemoryContext(new QueryMemoryReservationHandler(this::updateSystemMemory, this::tryReserveMemoryNotSupported), 0L)); + newRootAggregatedMemoryContext(new QueryMemoryReservationHandler(this::updateUserMemory, this::tryUpdateUserMemory), guaranteedMemory), + newRootAggregatedMemoryContext(new QueryMemoryReservationHandler(this::updateRevocableMemory, this::tryReserveMemoryNotSupported), 0L)); } public boolean isMemoryLimitsInitialized() @@ -131,21 +150,18 @@ public boolean isMemoryLimitsInitialized() } // TODO: This method should be removed, and the correct limit set in the constructor. However, due to the way QueryContext is constructed the memory limit is not known in advance - public synchronized void initializeMemoryLimits(boolean resourceOverCommit, long maxUserMemory, long maxTotalMemory, Optional maxTaskMemory) + public synchronized void initializeMemoryLimits(boolean resourceOverCommit, long maxUserMemory, Optional maxTaskMemory) { checkArgument(maxUserMemory >= 0, "maxUserMemory must be >= 0, found: %s", maxUserMemory); - checkArgument(maxTotalMemory >= 0, "maxTotalMemory must be >= 0, found: %s", maxTotalMemory); this.resourceOverCommit = resourceOverCommit; if (resourceOverCommit) { // Allow the query to use the entire pool. This way the worker will kill the query, if it uses the entire local memory pool. // The coordinator will kill the query if the cluster runs out of memory. this.maxUserMemory = memoryPool.getMaxBytes(); - this.maxTotalMemory = memoryPool.getMaxBytes(); this.maxTaskMemory = Optional.empty(); // disabled } else { this.maxUserMemory = maxUserMemory; - this.maxTotalMemory = maxTotalMemory; this.maxTaskMemory = maxTaskMemory; } memoryLimitsInitialized = true; @@ -163,23 +179,11 @@ public synchronized long getMaxUserMemory() return maxUserMemory; } - @VisibleForTesting - public synchronized long getMaxTotalMemory() - { - return maxTotalMemory; - } - public QueryId getQueryId() { return queryId; } - /** - * Deadlock is possible for concurrent user and system allocations when updateSystemMemory()/updateUserMemory - * calls queryMemoryContext.getUserMemory()/queryMemoryContext.getSystemMemory(), respectively. - * - * @see this#updateSystemMemory(String, long) for details. - */ private synchronized ListenableFuture updateUserMemory(String allocationTag, long delta) { if (delta >= 0) { @@ -210,39 +214,6 @@ private synchronized ListenableFuture updateRevocableMemory(String allocat return NOT_BLOCKED; } - private synchronized ListenableFuture updateSystemMemory(String allocationTag, long delta) - { - // We call memoryPool.getQueryMemoryReservation(queryId) instead of calling queryMemoryContext.getUserMemory() to - // calculate the total memory size. - // - // Calling the latter can result in a deadlock: - // * A thread doing a user allocation will acquire locks in this order: - // 1. monitor of queryMemoryContext.userAggregateMemoryContext - // 2. monitor of this (QueryContext) - // * The current thread doing a system allocation will acquire locks in this order: - // 1. monitor of this (QueryContext) - // 2. monitor of queryMemoryContext.userAggregateMemoryContext - - // Deadlock is possible for concurrent user and system allocations when updateSystemMemory()/updateUserMemory - // calls queryMemoryContext.getUserMemory()/queryMemoryContext.getSystemMemory(), respectively. For concurrent - // allocations of the same type (e.g., tryUpdateUserMemory/updateUserMemory) it is not possible as they share - // the same RootAggregatedMemoryContext instance, and one of the threads will be blocked on the monitor of that - // RootAggregatedMemoryContext instance even before calling the QueryContext methods (the monitors of - // RootAggregatedMemoryContext instance and this will be acquired in the same order). - long totalMemory = memoryPool.getQueryMemoryReservation(queryId); - if (delta >= 0) { - enforceTotalMemoryLimit(totalMemory, delta, maxTotalMemory); - ListenableFuture future = memoryPool.reserve(queryId, allocationTag, delta); - if (future.isDone()) { - return NOT_BLOCKED; - } - - return whenAnyComplete(ImmutableList.of(future, memoryPoolMoveFuture)); - } - memoryPool.free(queryId, allocationTag, -delta); - return NOT_BLOCKED; - } - //TODO move spill tracking to the new memory tracking framework public synchronized ListenableFuture reserveSpill(long bytes) { @@ -303,7 +274,6 @@ public synchronized void setMemoryPool(MemoryPool newMemoryPool) if (resourceOverCommit) { // Reset the memory limits based on the new pool assignment maxUserMemory = memoryPool.getMaxBytes(); - maxTotalMemory = memoryPool.getMaxBytes(); } // Create a new move future for allocations in new memory pool @@ -403,14 +373,6 @@ private void enforceUserMemoryLimit(long allocated, long delta, long maxMemory) } } - @GuardedBy("this") - private void enforceTotalMemoryLimit(long allocated, long delta, long maxMemory) - { - if (allocated + delta > maxMemory) { - throw exceededLocalTotalMemoryLimit(succinctBytes(maxMemory), getAdditionalFailureInfo(allocated, delta)); - } - } - @GuardedBy("this") private String getAdditionalFailureInfo(long allocated, long delta) { diff --git a/core/trino-main/src/main/java/io/trino/operator/AggregationOperator.java b/core/trino-main/src/main/java/io/trino/operator/AggregationOperator.java index dab720b2b57d..042b638fb981 100644 --- a/core/trino-main/src/main/java/io/trino/operator/AggregationOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/AggregationOperator.java @@ -41,15 +41,13 @@ public static class AggregationOperatorFactory private final int operatorId; private final PlanNodeId planNodeId; private final List aggregatorFactories; - private final boolean useSystemMemory; private boolean closed; - public AggregationOperatorFactory(int operatorId, PlanNodeId planNodeId, List aggregatorFactories, boolean useSystemMemory) + public AggregationOperatorFactory(int operatorId, PlanNodeId planNodeId, List aggregatorFactories) { this.operatorId = operatorId; this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); this.aggregatorFactories = ImmutableList.copyOf(aggregatorFactories); - this.useSystemMemory = useSystemMemory; } @Override @@ -57,7 +55,7 @@ public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, AggregationOperator.class.getSimpleName()); - return new AggregationOperator(operatorContext, aggregatorFactories, useSystemMemory); + return new AggregationOperator(operatorContext, aggregatorFactories); } @Override @@ -69,7 +67,7 @@ public void noMoreOperators() @Override public OperatorFactory duplicate() { - return new AggregationOperatorFactory(operatorId, planNodeId, aggregatorFactories, useSystemMemory); + return new AggregationOperatorFactory(operatorId, planNodeId, aggregatorFactories); } } @@ -81,19 +79,15 @@ private enum State } private final OperatorContext operatorContext; - private final LocalMemoryContext systemMemoryContext; private final LocalMemoryContext userMemoryContext; private final List aggregates; - private final boolean useSystemMemory; private State state = State.NEEDS_INPUT; - public AggregationOperator(OperatorContext operatorContext, List aggregatorFactories, boolean useSystemMemory) + public AggregationOperator(OperatorContext operatorContext, List aggregatorFactories) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); - this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(AggregationOperator.class.getSimpleName()); this.userMemoryContext = operatorContext.localUserMemoryContext(); - this.useSystemMemory = useSystemMemory; aggregates = aggregatorFactories.stream() .map(AggregatorFactory::createAggregator) @@ -118,7 +112,6 @@ public void finish() public void close() { userMemoryContext.setBytes(0); - systemMemoryContext.close(); } @Override @@ -144,12 +137,7 @@ public void addInput(Page page) aggregate.processPage(page); memorySize += aggregate.getEstimatedSize(); } - if (useSystemMemory) { - systemMemoryContext.setBytes(memorySize); - } - else { - userMemoryContext.setBytes(memorySize); - } + userMemoryContext.setBytes(memorySize); } @Override diff --git a/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClient.java b/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClient.java index 651458e500e8..971bd526b56e 100644 --- a/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClient.java +++ b/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClient.java @@ -77,7 +77,7 @@ public class DirectExchangeClient private final AtomicBoolean closed = new AtomicBoolean(); - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; private final Executor pageBufferClientCallbackExecutor; private final TaskFailureListener taskFailureListener; @@ -93,7 +93,7 @@ public DirectExchangeClient( boolean acknowledgePages, HttpClient httpClient, ScheduledExecutorService scheduledExecutor, - LocalMemoryContext systemMemoryContext, + LocalMemoryContext memoryContext, Executor pageBufferClientCallbackExecutor, TaskFailureListener taskFailureListener) { @@ -106,7 +106,7 @@ public DirectExchangeClient( this.acknowledgePages = acknowledgePages; this.httpClient = httpClient; this.scheduledExecutor = scheduledExecutor; - this.systemMemoryContext = systemMemoryContext; + this.memoryContext = memoryContext; this.pageBufferClientCallbackExecutor = requireNonNull(pageBufferClientCallbackExecutor, "pageBufferClientCallbackExecutor is null"); this.taskFailureListener = requireNonNull(taskFailureListener, "taskFailureListener is null"); } @@ -217,7 +217,7 @@ public Slice pollPage() return null; } - systemMemoryContext.setBytes(buffer.getRetainedSizeInBytes()); + memoryContext.setBytes(buffer.getRetainedSizeInBytes()); scheduleRequestIfNecessary(); return page; @@ -239,7 +239,7 @@ public synchronized void close() closeQuietly(client); } buffer.close(); - systemMemoryContext.setBytes(0); + memoryContext.setBytes(0); } private synchronized void scheduleRequestIfNecessary() @@ -296,7 +296,7 @@ private boolean addPages(HttpPageBufferClient client, List pages) // add pages outside of the lock if (!pages.isEmpty()) { buffer.addPages(client.getRemoteTaskId(), pages); - systemMemoryContext.setBytes(buffer.getRetainedSizeInBytes()); + memoryContext.setBytes(buffer.getRetainedSizeInBytes()); } return true; diff --git a/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClientFactory.java b/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClientFactory.java index 806e4072c976..716ae80da4c9 100644 --- a/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClientFactory.java +++ b/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClientFactory.java @@ -123,7 +123,7 @@ public ThreadPoolExecutorMBean getExecutor() } @Override - public DirectExchangeClient get(LocalMemoryContext systemMemoryContext, TaskFailureListener taskFailureListener, RetryPolicy retryPolicy) + public DirectExchangeClient get(LocalMemoryContext memoryContext, TaskFailureListener taskFailureListener, RetryPolicy retryPolicy) { DirectExchangeBuffer buffer; switch (retryPolicy) { @@ -148,7 +148,7 @@ public DirectExchangeClient get(LocalMemoryContext systemMemoryContext, TaskFail acknowledgePages, httpClient, scheduler, - systemMemoryContext, + memoryContext, pageBufferClientCallbackExecutor, taskFailureListener); } diff --git a/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClientSupplier.java b/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClientSupplier.java index a74f5a3e28db..3f389d3810c7 100644 --- a/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClientSupplier.java +++ b/core/trino-main/src/main/java/io/trino/operator/DirectExchangeClientSupplier.java @@ -18,5 +18,5 @@ public interface DirectExchangeClientSupplier { - DirectExchangeClient get(LocalMemoryContext systemMemoryContext, TaskFailureListener taskFailureListener, RetryPolicy retryPolicy); + DirectExchangeClient get(LocalMemoryContext memoryContext, TaskFailureListener taskFailureListener, RetryPolicy retryPolicy); } diff --git a/core/trino-main/src/main/java/io/trino/operator/Driver.java b/core/trino-main/src/main/java/io/trino/operator/Driver.java index 5fbd2bba6009..04a50cccd686 100644 --- a/core/trino-main/src/main/java/io/trino/operator/Driver.java +++ b/core/trino-main/src/main/java/io/trino/operator/Driver.java @@ -519,9 +519,6 @@ private void destroyIfNecessary() if (driverContext.getMemoryUsage() > 0) { log.error("Driver still has memory reserved after freeing all operator memory."); } - if (driverContext.getSystemMemoryUsage() > 0) { - log.error("Driver still has system memory reserved after freeing all operator memory."); - } if (driverContext.getRevocableMemoryUsage() > 0) { log.error("Driver still has revocable memory reserved after freeing all operator memory. Freeing it."); } diff --git a/core/trino-main/src/main/java/io/trino/operator/DriverContext.java b/core/trino-main/src/main/java/io/trino/operator/DriverContext.java index 05d92768067f..a7687775fa1c 100644 --- a/core/trino-main/src/main/java/io/trino/operator/DriverContext.java +++ b/core/trino-main/src/main/java/io/trino/operator/DriverContext.java @@ -213,11 +213,6 @@ public DriverYieldSignal getYieldSignal() return yieldSignal; } - public long getSystemMemoryUsage() - { - return driverMemoryContext.getSystemMemory(); - } - public long getMemoryUsage() { return driverMemoryContext.getUserMemory(); @@ -404,7 +399,6 @@ public DriverStats getDriverStats() elapsedTime.convertToMostSuccinctTimeUnit(), succinctBytes(driverMemoryContext.getUserMemory()), succinctBytes(driverMemoryContext.getRevocableMemory()), - succinctBytes(driverMemoryContext.getSystemMemory()), new Duration(totalScheduledTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalCpuTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalBlockedTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), diff --git a/core/trino-main/src/main/java/io/trino/operator/DriverStats.java b/core/trino-main/src/main/java/io/trino/operator/DriverStats.java index 757850fa3dc1..58615248a8ec 100644 --- a/core/trino-main/src/main/java/io/trino/operator/DriverStats.java +++ b/core/trino-main/src/main/java/io/trino/operator/DriverStats.java @@ -46,7 +46,6 @@ public class DriverStats private final DataSize userMemoryReservation; private final DataSize revocableMemoryReservation; - private final DataSize systemMemoryReservation; private final Duration totalScheduledTime; private final Duration totalCpuTime; @@ -88,7 +87,6 @@ public DriverStats() this.userMemoryReservation = DataSize.ofBytes(0); this.revocableMemoryReservation = DataSize.ofBytes(0); - this.systemMemoryReservation = DataSize.ofBytes(0); this.totalScheduledTime = new Duration(0, MILLISECONDS); this.totalCpuTime = new Duration(0, MILLISECONDS); @@ -131,7 +129,6 @@ public DriverStats( @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation, - @JsonProperty("systemMemoryReservation") DataSize systemMemoryReservation, @JsonProperty("totalScheduledTime") Duration totalScheduledTime, @JsonProperty("totalCpuTime") Duration totalCpuTime, @@ -171,7 +168,6 @@ public DriverStats( this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null"); this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null"); - this.systemMemoryReservation = requireNonNull(systemMemoryReservation, "systemMemoryReservation is null"); this.totalScheduledTime = requireNonNull(totalScheduledTime, "totalScheduledTime is null"); this.totalCpuTime = requireNonNull(totalCpuTime, "totalCpuTime is null"); @@ -257,12 +253,6 @@ public DataSize getRevocableMemoryReservation() return revocableMemoryReservation; } - @JsonProperty - public DataSize getSystemMemoryReservation() - { - return systemMemoryReservation; - } - @JsonProperty public Duration getTotalScheduledTime() { diff --git a/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java b/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java index bba0077801f3..4d6e3d443fac 100644 --- a/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java @@ -94,7 +94,7 @@ public SourceOperator createOperator(DriverContext driverContext) checkState(!closed, "Factory is already closed"); TaskContext taskContext = driverContext.getPipelineContext().getTaskContext(); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, sourceId, ExchangeOperator.class.getSimpleName()); - LocalMemoryContext memoryContext = driverContext.getPipelineContext().localSystemMemoryContext(); + LocalMemoryContext memoryContext = driverContext.getPipelineContext().localMemoryContext(); if (exchangeDataSource == null) { // The decision of what exchange to use (streaming vs external) is currently made at the scheduling phase. It is more convenient to deliver it as part of a RemoteSplit. // Postponing this decision until scheduling allows to dynamically change the exchange type as part of an adaptive query re-planning. diff --git a/core/trino-main/src/main/java/io/trino/operator/FilterAndProjectOperator.java b/core/trino-main/src/main/java/io/trino/operator/FilterAndProjectOperator.java index ab91191e99e2..e30cfb8c7fdc 100644 --- a/core/trino-main/src/main/java/io/trino/operator/FilterAndProjectOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/FilterAndProjectOperator.java @@ -63,7 +63,7 @@ private FilterAndProjectOperator( page, avoidPageMaterialization)) .transformProcessor(processor -> mergePages(types, minOutputPageSize.toBytes(), minOutputPageRowCount, processor, localAggregatedMemoryContext)) - .blocking(() -> memoryTrackingContext.localSystemMemoryContext().setBytes(localAggregatedMemoryContext.getBytes())); + .blocking(() -> memoryTrackingContext.localUserMemoryContext().setBytes(localAggregatedMemoryContext.getBytes())); } @Override diff --git a/core/trino-main/src/main/java/io/trino/operator/HashAggregationOperator.java b/core/trino-main/src/main/java/io/trino/operator/HashAggregationOperator.java index f2e3d735e29b..4d7060536622 100644 --- a/core/trino-main/src/main/java/io/trino/operator/HashAggregationOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/HashAggregationOperator.java @@ -37,7 +37,6 @@ import java.util.Optional; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Verify.verify; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static io.trino.operator.aggregation.builder.InMemoryHashAggregationBuilder.toTypes; import static io.trino.sql.planner.optimizations.HashGenerationOptimizer.INITIAL_HASH_VALUE; @@ -71,7 +70,6 @@ public static class HashAggregationOperatorFactory private final SpillerFactory spillerFactory; private final JoinCompiler joinCompiler; private final BlockTypeOperators blockTypeOperators; - private final boolean useSystemMemory; private boolean closed; @@ -89,8 +87,7 @@ public HashAggregationOperatorFactory( int expectedGroups, Optional maxPartialMemory, JoinCompiler joinCompiler, - BlockTypeOperators blockTypeOperators, - boolean useSystemMemory) + BlockTypeOperators blockTypeOperators) { this(operatorId, planNodeId, @@ -111,8 +108,7 @@ public HashAggregationOperatorFactory( throw new UnsupportedOperationException(); }, joinCompiler, - blockTypeOperators, - useSystemMemory); + blockTypeOperators); } public HashAggregationOperatorFactory( @@ -132,8 +128,7 @@ public HashAggregationOperatorFactory( DataSize unspillMemoryLimit, SpillerFactory spillerFactory, JoinCompiler joinCompiler, - BlockTypeOperators blockTypeOperators, - boolean useSystemMemory) + BlockTypeOperators blockTypeOperators) { this(operatorId, planNodeId, @@ -152,8 +147,7 @@ public HashAggregationOperatorFactory( DataSize.succinctBytes((long) (unspillMemoryLimit.toBytes() * MERGE_WITH_MEMORY_RATIO)), spillerFactory, joinCompiler, - blockTypeOperators, - useSystemMemory); + blockTypeOperators); } @VisibleForTesting @@ -175,8 +169,7 @@ public HashAggregationOperatorFactory( DataSize memoryLimitForMergeWithMemory, SpillerFactory spillerFactory, JoinCompiler joinCompiler, - BlockTypeOperators blockTypeOperators, - boolean useSystemMemory) + BlockTypeOperators blockTypeOperators) { this.operatorId = operatorId; this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); @@ -196,7 +189,6 @@ public HashAggregationOperatorFactory( this.spillerFactory = requireNonNull(spillerFactory, "spillerFactory is null"); this.joinCompiler = requireNonNull(joinCompiler, "joinCompiler is null"); this.blockTypeOperators = requireNonNull(blockTypeOperators, "blockTypeOperators is null"); - this.useSystemMemory = useSystemMemory; } @Override @@ -222,8 +214,7 @@ public Operator createOperator(DriverContext driverContext) memoryLimitForMergeWithMemory, spillerFactory, joinCompiler, - blockTypeOperators, - useSystemMemory); + blockTypeOperators); return hashAggregationOperator; } @@ -254,8 +245,7 @@ public OperatorFactory duplicate() memoryLimitForMergeWithMemory, spillerFactory, joinCompiler, - blockTypeOperators, - useSystemMemory); + blockTypeOperators); } } @@ -276,13 +266,12 @@ public OperatorFactory duplicate() private final SpillerFactory spillerFactory; private final JoinCompiler joinCompiler; private final BlockTypeOperators blockTypeOperators; - private final boolean useSystemMemory; private final List types; private final HashCollisionsCounter hashCollisionsCounter; private HashAggregationBuilder aggregationBuilder; - private LocalMemoryContext memoryContext; + private final LocalMemoryContext memoryContext; private WorkProcessor outputPages; private boolean inputProcessed; private boolean finishing; @@ -308,8 +297,7 @@ private HashAggregationOperator( DataSize memoryLimitForMergeWithMemory, SpillerFactory spillerFactory, JoinCompiler joinCompiler, - BlockTypeOperators blockTypeOperators, - boolean useSystemMemory) + BlockTypeOperators blockTypeOperators) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); requireNonNull(step, "step is null"); @@ -335,12 +323,8 @@ private HashAggregationOperator( this.blockTypeOperators = requireNonNull(blockTypeOperators, "blockTypeOperators is null"); this.hashCollisionsCounter = new HashCollisionsCounter(operatorContext); operatorContext.setInfoSupplier(hashCollisionsCounter); - this.useSystemMemory = useSystemMemory; this.memoryContext = operatorContext.localUserMemoryContext(); - if (useSystemMemory) { - this.memoryContext = operatorContext.localSystemMemoryContext(); - } } @Override @@ -407,7 +391,6 @@ public void addInput(Page page) }); } else { - verify(!useSystemMemory, "using system memory in spillable aggregations is not supported"); aggregationBuilder = new SpillableHashAggregationBuilder( aggregatorFactories, step, diff --git a/core/trino-main/src/main/java/io/trino/operator/MergeOperator.java b/core/trino-main/src/main/java/io/trino/operator/MergeOperator.java index e2072c8b8d15..81acafd552b8 100644 --- a/core/trino-main/src/main/java/io/trino/operator/MergeOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/MergeOperator.java @@ -160,7 +160,7 @@ public Supplier> addSplit(Split split) checkState(!blockedOnSplits.isDone(), "noMoreSplits has been called already"); TaskContext taskContext = operatorContext.getDriverContext().getPipelineContext().getTaskContext(); - DirectExchangeClient client = closer.register(directExchangeClientSupplier.get(operatorContext.localSystemMemoryContext(), taskContext::sourceTaskFailed, RetryPolicy.NONE)); + DirectExchangeClient client = closer.register(directExchangeClientSupplier.get(operatorContext.localUserMemoryContext(), taskContext::sourceTaskFailed, RetryPolicy.NONE)); RemoteSplit remoteSplit = (RemoteSplit) split.getConnectorSplit(); // Only fault tolerant execution mode is expected to execute external exchanges. // MergeOperator is used for distributed sort only and it is not compatible (and disabled) with fault tolerant execution mode. diff --git a/core/trino-main/src/main/java/io/trino/operator/OperatorContext.java b/core/trino-main/src/main/java/io/trino/operator/OperatorContext.java index 2104be8cfbf0..f21b1f81b4f9 100644 --- a/core/trino-main/src/main/java/io/trino/operator/OperatorContext.java +++ b/core/trino-main/src/main/java/io/trino/operator/OperatorContext.java @@ -106,7 +106,6 @@ public class OperatorContext private final AtomicReference>> nestedOperatorStatsSupplier = new AtomicReference<>(); private final AtomicLong peakUserMemoryReservation = new AtomicLong(); - private final AtomicLong peakSystemMemoryReservation = new AtomicLong(); private final AtomicLong peakRevocableMemoryReservation = new AtomicLong(); private final AtomicLong peakTotalMemoryReservation = new AtomicLong(); @@ -278,9 +277,9 @@ public ListenableFuture isWaitingForRevocableMemory() } // caller should close this context as it's a new context - public LocalMemoryContext newLocalSystemMemoryContext(String allocationTag) + public LocalMemoryContext newLocalUserMemoryContext(String allocationTag) { - return new InternalLocalMemoryContext(operatorMemoryContext.newSystemMemoryContext(allocationTag), memoryFuture, this::updatePeakMemoryReservations, true); + return new InternalLocalMemoryContext(operatorMemoryContext.newUserMemoryContext(allocationTag), memoryFuture, this::updatePeakMemoryReservations, true); } // caller shouldn't close this context as it's managed by the OperatorContext @@ -289,12 +288,6 @@ public LocalMemoryContext localUserMemoryContext() return new InternalLocalMemoryContext(operatorMemoryContext.localUserMemoryContext(), memoryFuture, this::updatePeakMemoryReservations, false); } - // caller shouldn't close this context as it's managed by the OperatorContext - public LocalMemoryContext localSystemMemoryContext() - { - return new InternalLocalMemoryContext(operatorMemoryContext.localSystemMemoryContext(), memoryFuture, this::updatePeakMemoryReservations, false); - } - // caller shouldn't close this context as it's managed by the OperatorContext public LocalMemoryContext localRevocableMemoryContext() { @@ -307,12 +300,6 @@ public AggregatedMemoryContext aggregateUserMemoryContext() return new InternalAggregatedMemoryContext(operatorMemoryContext.aggregateUserMemoryContext(), memoryFuture, this::updatePeakMemoryReservations, false); } - // caller shouldn't close this context as it's managed by the OperatorContext - public AggregatedMemoryContext aggregateSystemMemoryContext() - { - return new InternalAggregatedMemoryContext(operatorMemoryContext.aggregateSystemMemoryContext(), memoryFuture, this::updatePeakMemoryReservations, false); - } - // caller shouldn't close this context as it's managed by the OperatorContext public AggregatedMemoryContext aggregateRevocableMemoryContext() { @@ -325,12 +312,6 @@ public AggregatedMemoryContext newAggregateUserMemoryContext() return new InternalAggregatedMemoryContext(operatorMemoryContext.newAggregateUserMemoryContext(), memoryFuture, this::updatePeakMemoryReservations, true); } - // caller should close this context as it's a new context - public AggregatedMemoryContext newAggregateSystemMemoryContext() - { - return new InternalAggregatedMemoryContext(operatorMemoryContext.newAggregateSystemMemoryContext(), memoryFuture, this::updatePeakMemoryReservations, true); - } - // caller should close this context as it's a new context public AggregatedMemoryContext newAggregateRevocableMemoryContext() { @@ -341,11 +322,12 @@ public AggregatedMemoryContext newAggregateRevocableMemoryContext() private void updatePeakMemoryReservations() { long userMemory = operatorMemoryContext.getUserMemory(); - long systemMemory = operatorMemoryContext.getSystemMemory(); long revocableMemory = operatorMemoryContext.getRevocableMemory(); - long totalMemory = userMemory + systemMemory; + // TODO on cluster level, "total memory" means "user memory + revocable", and used to include the system memory. + // Here, the total memory used to be user+system, and sans revocable. This apparent inconsistency should be removed. + // Perhaps, we don't need to track "total memory" here. + long totalMemory = userMemory; peakUserMemoryReservation.accumulateAndGet(userMemory, Math::max); - peakSystemMemoryReservation.accumulateAndGet(systemMemory, Math::max); peakRevocableMemoryReservation.accumulateAndGet(revocableMemory, Math::max); peakTotalMemoryReservation.accumulateAndGet(totalMemory, Math::max); } @@ -396,10 +378,6 @@ public void destroy() operatorMemoryContext.close(); - if (operatorMemoryContext.getSystemMemory() != 0) { - throw new TrinoException(GENERIC_INTERNAL_ERROR, format("Operator %s has non-zero system memory (%d bytes) after destroy()", this, operatorMemoryContext.getSystemMemory())); - } - if (operatorMemoryContext.getUserMemory() != 0) { throw new TrinoException(GENERIC_INTERNAL_ERROR, format("Operator %s has non-zero user memory (%d bytes) after destroy()", this, operatorMemoryContext.getUserMemory())); } @@ -592,10 +570,8 @@ private OperatorStats getOperatorStats() DataSize.ofBytes(operatorMemoryContext.getUserMemory()), DataSize.ofBytes(getReservedRevocableBytes()), - DataSize.ofBytes(operatorMemoryContext.getSystemMemory()), DataSize.ofBytes(peakUserMemoryReservation.get()), - DataSize.ofBytes(peakSystemMemoryReservation.get()), DataSize.ofBytes(peakRevocableMemoryReservation.get()), DataSize.ofBytes(peakTotalMemoryReservation.get()), diff --git a/core/trino-main/src/main/java/io/trino/operator/OperatorStats.java b/core/trino-main/src/main/java/io/trino/operator/OperatorStats.java index bd360910424c..58b55dd40cb2 100644 --- a/core/trino-main/src/main/java/io/trino/operator/OperatorStats.java +++ b/core/trino-main/src/main/java/io/trino/operator/OperatorStats.java @@ -76,9 +76,7 @@ public class OperatorStats private final DataSize userMemoryReservation; private final DataSize revocableMemoryReservation; - private final DataSize systemMemoryReservation; private final DataSize peakUserMemoryReservation; - private final DataSize peakSystemMemoryReservation; private final DataSize peakRevocableMemoryReservation; private final DataSize peakTotalMemoryReservation; @@ -131,9 +129,7 @@ public OperatorStats( @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation, - @JsonProperty("systemMemoryReservation") DataSize systemMemoryReservation, @JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation, - @JsonProperty("peakSystemMemoryReservation") DataSize peakSystemMemoryReservation, @JsonProperty("peakRevocableMemoryReservation") DataSize peakRevocableMemoryReservation, @JsonProperty("peakTotalMemoryReservation") DataSize peakTotalMemoryReservation, @@ -188,10 +184,8 @@ public OperatorStats( this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null"); this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null"); - this.systemMemoryReservation = requireNonNull(systemMemoryReservation, "systemMemoryReservation is null"); this.peakUserMemoryReservation = requireNonNull(peakUserMemoryReservation, "peakUserMemoryReservation is null"); - this.peakSystemMemoryReservation = requireNonNull(peakSystemMemoryReservation, "peakSystemMemoryReservation is null"); this.peakRevocableMemoryReservation = requireNonNull(peakRevocableMemoryReservation, "peakRevocableMemoryReservation is null"); this.peakTotalMemoryReservation = requireNonNull(peakTotalMemoryReservation, "peakTotalMemoryReservation is null"); @@ -394,12 +388,6 @@ public DataSize getRevocableMemoryReservation() return revocableMemoryReservation; } - @JsonProperty - public DataSize getSystemMemoryReservation() - { - return systemMemoryReservation; - } - @JsonProperty public DataSize getPeakUserMemoryReservation() { @@ -412,12 +400,6 @@ public DataSize getPeakRevocableMemoryReservation() return peakRevocableMemoryReservation; } - @JsonProperty - public DataSize getPeakSystemMemoryReservation() - { - return peakSystemMemoryReservation; - } - @JsonProperty public DataSize getPeakTotalMemoryReservation() { @@ -484,9 +466,7 @@ public OperatorStats add(Iterable operators) long memoryReservation = this.userMemoryReservation.toBytes(); long revocableMemoryReservation = this.revocableMemoryReservation.toBytes(); - long systemMemoryReservation = this.systemMemoryReservation.toBytes(); long peakUserMemory = this.peakUserMemoryReservation.toBytes(); - long peakSystemMemory = this.peakSystemMemoryReservation.toBytes(); long peakRevocableMemory = this.peakRevocableMemoryReservation.toBytes(); long peakTotalMemory = this.peakTotalMemoryReservation.toBytes(); @@ -533,10 +513,8 @@ public OperatorStats add(Iterable operators) memoryReservation += operator.getUserMemoryReservation().toBytes(); revocableMemoryReservation += operator.getRevocableMemoryReservation().toBytes(); - systemMemoryReservation += operator.getSystemMemoryReservation().toBytes(); peakUserMemory = max(peakUserMemory, operator.getPeakUserMemoryReservation().toBytes()); - peakSystemMemory = max(peakSystemMemory, operator.getPeakSystemMemoryReservation().toBytes()); peakRevocableMemory = max(peakRevocableMemory, operator.getPeakRevocableMemoryReservation().toBytes()); peakTotalMemory = max(peakTotalMemory, operator.getPeakTotalMemoryReservation().toBytes()); @@ -594,9 +572,7 @@ public OperatorStats add(Iterable operators) DataSize.ofBytes(memoryReservation), DataSize.ofBytes(revocableMemoryReservation), - DataSize.ofBytes(systemMemoryReservation), DataSize.ofBytes(peakUserMemory), - DataSize.ofBytes(peakSystemMemory), DataSize.ofBytes(peakRevocableMemory), DataSize.ofBytes(peakTotalMemory), @@ -662,9 +638,7 @@ public OperatorStats summarize() finishCpu, userMemoryReservation, revocableMemoryReservation, - systemMemoryReservation, peakUserMemoryReservation, - peakSystemMemoryReservation, peakRevocableMemoryReservation, peakTotalMemoryReservation, spilledDataSize, diff --git a/core/trino-main/src/main/java/io/trino/operator/OrderByOperator.java b/core/trino-main/src/main/java/io/trino/operator/OrderByOperator.java index 7097f0a48536..02254b6a0f78 100644 --- a/core/trino-main/src/main/java/io/trino/operator/OrderByOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/OrderByOperator.java @@ -312,7 +312,7 @@ private ListenableFuture spillToDisk() spiller = Optional.of(spillerFactory.get().create( sourceTypes, operatorContext.getSpillContext(), - operatorContext.newAggregateSystemMemoryContext())); + operatorContext.newAggregateUserMemoryContext())); } pageIndex.sort(sortChannels, sortOrder); diff --git a/core/trino-main/src/main/java/io/trino/operator/PipelineContext.java b/core/trino-main/src/main/java/io/trino/operator/PipelineContext.java index 0e8a6672a180..97b76a94d1f8 100644 --- a/core/trino-main/src/main/java/io/trino/operator/PipelineContext.java +++ b/core/trino-main/src/main/java/io/trino/operator/PipelineContext.java @@ -256,9 +256,9 @@ public synchronized void freeSpill(long bytes) taskContext.freeSpill(bytes); } - public LocalMemoryContext localSystemMemoryContext() + public LocalMemoryContext localMemoryContext() { - return pipelineMemoryContext.localSystemMemoryContext(); + return pipelineMemoryContext.localUserMemoryContext(); } public boolean isPerOperatorCpuTimerEnabled() @@ -466,7 +466,6 @@ public PipelineStats getPipelineStats() succinctBytes(pipelineMemoryContext.getUserMemory()), succinctBytes(pipelineMemoryContext.getRevocableMemory()), - succinctBytes(pipelineMemoryContext.getSystemMemory()), queuedTime.snapshot(), elapsedTime.snapshot(), diff --git a/core/trino-main/src/main/java/io/trino/operator/PipelineStats.java b/core/trino-main/src/main/java/io/trino/operator/PipelineStats.java index dd76e07b10ca..8207a81bda5f 100644 --- a/core/trino-main/src/main/java/io/trino/operator/PipelineStats.java +++ b/core/trino-main/src/main/java/io/trino/operator/PipelineStats.java @@ -55,7 +55,6 @@ public class PipelineStats private final DataSize userMemoryReservation; private final DataSize revocableMemoryReservation; - private final DataSize systemMemoryReservation; private final DistributionSnapshot queuedTime; private final DistributionSnapshot elapsedTime; @@ -110,7 +109,6 @@ public PipelineStats( @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation, - @JsonProperty("systemMemoryReservation") DataSize systemMemoryReservation, @JsonProperty("queuedTime") DistributionSnapshot queuedTime, @JsonProperty("elapsedTime") DistributionSnapshot elapsedTime, @@ -172,7 +170,6 @@ public PipelineStats( this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null"); this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null"); - this.systemMemoryReservation = requireNonNull(systemMemoryReservation, "systemMemoryReservation is null"); this.queuedTime = requireNonNull(queuedTime, "queuedTime is null"); this.elapsedTime = requireNonNull(elapsedTime, "elapsedTime is null"); @@ -315,12 +312,6 @@ public DataSize getRevocableMemoryReservation() return revocableMemoryReservation; } - @JsonProperty - public DataSize getSystemMemoryReservation() - { - return systemMemoryReservation; - } - @JsonProperty public DistributionSnapshot getQueuedTime() { @@ -467,7 +458,6 @@ public PipelineStats summarize() completedDrivers, userMemoryReservation, revocableMemoryReservation, - systemMemoryReservation, queuedTime, elapsedTime, totalScheduledTime, diff --git a/core/trino-main/src/main/java/io/trino/operator/ScanFilterAndProjectOperator.java b/core/trino-main/src/main/java/io/trino/operator/ScanFilterAndProjectOperator.java index d84484ef940e..cc88991fcce8 100644 --- a/core/trino-main/src/main/java/io/trino/operator/ScanFilterAndProjectOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/ScanFilterAndProjectOperator.java @@ -111,7 +111,7 @@ private ScanFilterAndProjectOperator( columns, dynamicFilter, types, - requireNonNull(memoryTrackingContext, "memoryTrackingContext is null").aggregateSystemMemoryContext(), + requireNonNull(memoryTrackingContext, "memoryTrackingContext is null").aggregateUserMemoryContext(), minOutputPageSize, minOutputPageRowCount, avoidPageMaterialization)); @@ -336,7 +336,7 @@ public ProcessState process() { if (!finished) { CursorProcessorOutput output = cursorProcessor.process(session, yieldSignal, cursor, pageBuilder); - pageSourceMemoryContext.setBytes(cursor.getSystemMemoryUsage()); + pageSourceMemoryContext.setBytes(cursor.getMemoryUsage()); processedPositions += output.getProcessedRows(); // TODO: derive better values for cursors @@ -390,7 +390,7 @@ public ProcessState process() } Page page = pageSource.getNextPage(); - pageSourceMemoryContext.setBytes(pageSource.getSystemMemoryUsage()); + pageSourceMemoryContext.setBytes(pageSource.getMemoryUsage()); if (page == null) { if (pageSource.isFinished()) { diff --git a/core/trino-main/src/main/java/io/trino/operator/StreamingAggregationOperator.java b/core/trino-main/src/main/java/io/trino/operator/StreamingAggregationOperator.java index 0a134323f061..08721a4c5b03 100644 --- a/core/trino-main/src/main/java/io/trino/operator/StreamingAggregationOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/StreamingAggregationOperator.java @@ -26,7 +26,6 @@ import io.trino.spi.block.Block; import io.trino.spi.type.Type; import io.trino.sql.gen.JoinCompiler; -import io.trino.sql.planner.plan.AggregationNode.Step; import io.trino.sql.planner.plan.PlanNodeId; import javax.annotation.Nullable; @@ -54,7 +53,6 @@ public static OperatorFactory createOperatorFactory( List sourceTypes, List groupByTypes, List groupByChannels, - Step step, List aggregatorFactories, JoinCompiler joinCompiler) { @@ -64,7 +62,6 @@ public static OperatorFactory createOperatorFactory( sourceTypes, groupByTypes, groupByChannels, - step, aggregatorFactories, joinCompiler)); } @@ -77,7 +74,6 @@ private static class Factory private final List sourceTypes; private final List groupByTypes; private final List groupByChannels; - private final Step step; private final List aggregatorFactories; private final JoinCompiler joinCompiler; private boolean closed; @@ -88,7 +84,6 @@ private Factory( List sourceTypes, List groupByTypes, List groupByChannels, - Step step, List aggregatorFactories, JoinCompiler joinCompiler) { @@ -97,7 +92,6 @@ private Factory( this.sourceTypes = ImmutableList.copyOf(requireNonNull(sourceTypes, "sourceTypes is null")); this.groupByTypes = ImmutableList.copyOf(requireNonNull(groupByTypes, "groupByTypes is null")); this.groupByChannels = ImmutableList.copyOf(requireNonNull(groupByChannels, "groupByChannels is null")); - this.step = step; this.aggregatorFactories = ImmutableList.copyOf(requireNonNull(aggregatorFactories, "aggregatorFactories is null")); this.joinCompiler = requireNonNull(joinCompiler, "joinCompiler is null"); } @@ -124,7 +118,7 @@ public String getOperatorType() public WorkProcessorOperator create(ProcessorContext processorContext, WorkProcessor sourcePages) { checkState(!closed, "Factory is already closed"); - return new StreamingAggregationOperator(processorContext, sourcePages, sourceTypes, groupByTypes, groupByChannels, step, aggregatorFactories, joinCompiler); + return new StreamingAggregationOperator(processorContext, sourcePages, sourceTypes, groupByTypes, groupByChannels, aggregatorFactories, joinCompiler); } @Override @@ -136,7 +130,7 @@ public void close() @Override public Factory duplicate() { - return new Factory(operatorId, planNodeId, sourceTypes, groupByTypes, groupByChannels, step, aggregatorFactories, joinCompiler); + return new Factory(operatorId, planNodeId, sourceTypes, groupByTypes, groupByChannels, aggregatorFactories, joinCompiler); } } @@ -148,7 +142,6 @@ private StreamingAggregationOperator( List sourceTypes, List groupByTypes, List groupByChannels, - Step step, List aggregatorFactories, JoinCompiler joinCompiler) { @@ -158,7 +151,6 @@ private StreamingAggregationOperator( sourceTypes, groupByTypes, groupByChannels, - step, aggregatorFactories, joinCompiler)); } @@ -172,12 +164,10 @@ public WorkProcessor getOutputPages() private static class StreamingAggregation implements Transformation { - private final LocalMemoryContext systemMemoryContext; private final LocalMemoryContext userMemoryContext; private final List groupByTypes; private final int[] groupByChannels; private final List aggregatorFactories; - private final Step step; private final PagesHashStrategy pagesHashStrategy; private List aggregates; @@ -190,17 +180,14 @@ private StreamingAggregation( List sourceTypes, List groupByTypes, List groupByChannels, - Step step, List aggregatorFactories, JoinCompiler joinCompiler) { requireNonNull(processorContext, "processorContext is null"); - this.systemMemoryContext = processorContext.getMemoryTrackingContext().localSystemMemoryContext(); this.userMemoryContext = processorContext.getMemoryTrackingContext().localUserMemoryContext(); this.groupByTypes = ImmutableList.copyOf(requireNonNull(groupByTypes, "groupByTypes is null")); this.groupByChannels = Ints.toArray(requireNonNull(groupByChannels, "groupByChannels is null")); this.aggregatorFactories = requireNonNull(aggregatorFactories, "aggregatorFactories is null"); - this.step = requireNonNull(step, "step is null"); this.aggregates = aggregatorFactories.stream() .map(AggregatorFactory::createAggregator) @@ -269,12 +256,7 @@ private void updateMemoryUsage() memorySize += currentGroup.getRetainedSizeInBytes(); } - if (step.isOutputPartial()) { - systemMemoryContext.setBytes(memorySize); - } - else { - userMemoryContext.setBytes(memorySize); - } + userMemoryContext.setBytes(memorySize); } private void processInput(Page page) diff --git a/core/trino-main/src/main/java/io/trino/operator/TableScanOperator.java b/core/trino-main/src/main/java/io/trino/operator/TableScanOperator.java index 1dd54319ae21..842336fdcfdc 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TableScanOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/TableScanOperator.java @@ -145,7 +145,7 @@ public void noMoreOperators() private final TableHandle table; private final List columns; private final DynamicFilter dynamicFilter; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; private final SettableFuture blocked = SettableFuture.create(); @Nullable @@ -173,7 +173,7 @@ public TableScanOperator( this.table = requireNonNull(table, "table is null"); this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.dynamicFilter = requireNonNull(dynamicFilter, "dynamicFilter is null"); - this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(TableScanOperator.class.getSimpleName()); + this.memoryContext = operatorContext.newLocalUserMemoryContext(TableScanOperator.class.getSimpleName()); } @Override @@ -247,7 +247,7 @@ public void finish() catch (IOException e) { throw new UncheckedIOException(e); } - systemMemoryContext.setBytes(source.getSystemMemoryUsage()); + memoryContext.setBytes(source.getMemoryUsage()); operatorContext.setLatestConnectorMetrics(source.getMetrics()); } } @@ -258,7 +258,7 @@ public boolean isFinished() if (!finished) { finished = (source != null) && source.isFinished(); if (source != null) { - systemMemoryContext.setBytes(source.getSystemMemoryUsage()); + memoryContext.setBytes(source.getMemoryUsage()); } } @@ -328,8 +328,8 @@ public Page getOutput() readTimeNanos = endReadTimeNanos; } - // updating system memory usage should happen after page is loaded. - systemMemoryContext.setBytes(source.getSystemMemoryUsage()); + // updating memory usage should happen after page is loaded. + memoryContext.setBytes(source.getMemoryUsage()); operatorContext.setLatestConnectorMetrics(source.getMetrics()); return page; } diff --git a/core/trino-main/src/main/java/io/trino/operator/TableScanWorkProcessorOperator.java b/core/trino-main/src/main/java/io/trino/operator/TableScanWorkProcessorOperator.java index 3506689026a5..2b80242bf056 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TableScanWorkProcessorOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/TableScanWorkProcessorOperator.java @@ -72,7 +72,7 @@ public TableScanWorkProcessorOperator( table, columns, dynamicFilter, - memoryTrackingContext.aggregateSystemMemoryContext()); + memoryTrackingContext.aggregateUserMemoryContext()); this.pages = splits.flatTransform(splitToPages); } @@ -196,7 +196,7 @@ public TransformationState> process(Split split) recordMaterializedBytes(page, sizeInBytes -> processedBytes += sizeInBytes); return page; }) - .blocking(() -> memoryContext.setBytes(source.getSystemMemoryUsage()))); + .blocking(() -> memoryContext.setBytes(source.getMemoryUsage()))); } Supplier> getUpdatablePageSourceSupplier() diff --git a/core/trino-main/src/main/java/io/trino/operator/TableWriterOperator.java b/core/trino-main/src/main/java/io/trino/operator/TableWriterOperator.java index bf84936f95b4..133843a16618 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TableWriterOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/TableWriterOperator.java @@ -187,7 +187,7 @@ public TableWriterOperator( boolean statisticsCpuTimerEnabled) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); - this.pageSinkMemoryContext = operatorContext.newLocalSystemMemoryContext(TableWriterOperator.class.getSimpleName()); + this.pageSinkMemoryContext = operatorContext.newLocalUserMemoryContext(TableWriterOperator.class.getSimpleName()); this.pageSink = requireNonNull(pageSink, "pageSink is null"); this.columnChannels = requireNonNull(columnChannels, "columnChannels is null"); this.notNullChannelColumnNames = requireNonNull(notNullChannelColumnNames, "notNullChannelColumnNames is null"); @@ -392,7 +392,7 @@ private void updateWrittenBytes() private void updateMemoryUsage() { - long pageSinkMemoryUsage = pageSink.getSystemMemoryUsage(); + long pageSinkMemoryUsage = pageSink.getMemoryUsage(); pageSinkMemoryContext.setBytes(pageSinkMemoryUsage); pageSinkPeakMemoryUsage.accumulateAndGet(pageSinkMemoryUsage, Math::max); } diff --git a/core/trino-main/src/main/java/io/trino/operator/TaskContext.java b/core/trino-main/src/main/java/io/trino/operator/TaskContext.java index 36a3fbf17e0e..d24964f1b305 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TaskContext.java +++ b/core/trino-main/src/main/java/io/trino/operator/TaskContext.java @@ -97,14 +97,10 @@ public class TaskContext private final Object cumulativeMemoryLock = new Object(); private final AtomicDouble cumulativeUserMemory = new AtomicDouble(0.0); - private final AtomicDouble cumulativeSystemMemory = new AtomicDouble(0.0); @GuardedBy("cumulativeMemoryLock") private long lastUserMemoryReservation; - @GuardedBy("cumulativeMemoryLock") - private long lastSystemMemoryReservation; - @GuardedBy("cumulativeMemoryLock") private long lastTaskStatCallNanos; @@ -170,8 +166,7 @@ private TaskContext( MemoryAllocationValidator memoryValidator = new TaskAllocationValidator(maxMemory.get()); this.taskMemoryContext = new MemoryTrackingContext( new ValidatingAggregateContext(taskMemoryContext.aggregateUserMemoryContext(), memoryValidator), - taskMemoryContext.aggregateRevocableMemoryContext(), - new ValidatingAggregateContext(taskMemoryContext.aggregateSystemMemoryContext(), memoryValidator)); + taskMemoryContext.aggregateRevocableMemoryContext()); } else { this.taskMemoryContext = taskMemoryContext; @@ -275,11 +270,6 @@ public DataSize getMemoryReservation() return DataSize.ofBytes(taskMemoryContext.getUserMemory()); } - public DataSize getSystemMemoryReservation() - { - return DataSize.ofBytes(taskMemoryContext.getSystemMemory()); - } - public DataSize getRevocableMemoryReservation() { return DataSize.ofBytes(taskMemoryContext.getRevocableMemory()); @@ -318,9 +308,9 @@ public synchronized void freeSpill(long bytes) queryContext.freeSpill(bytes); } - public LocalMemoryContext localSystemMemoryContext() + public LocalMemoryContext localMemoryContext() { - return taskMemoryContext.localSystemMemoryContext(); + return taskMemoryContext.localUserMemoryContext(); } public boolean isPerOperatorCpuTimerEnabled() @@ -532,19 +522,15 @@ public TaskStats getTaskStats() Duration fullGcTime = getFullGcTime(); long userMemory = taskMemoryContext.getUserMemory(); - long systemMemory = taskMemoryContext.getSystemMemory(); synchronized (cumulativeMemoryLock) { long currentTimeNanos = System.nanoTime(); double sinceLastPeriodMillis = (currentTimeNanos - lastTaskStatCallNanos) / 1_000_000.0; long averageUserMemoryForLastPeriod = (userMemory + lastUserMemoryReservation) / 2; cumulativeUserMemory.addAndGet(averageUserMemoryForLastPeriod * sinceLastPeriodMillis); - long averageSystemMemoryForLastPeriod = (systemMemory + lastSystemMemoryReservation) / 2; - cumulativeSystemMemory.addAndGet(averageSystemMemoryForLastPeriod * sinceLastPeriodMillis); lastTaskStatCallNanos = currentTimeNanos; lastUserMemoryReservation = userMemory; - lastSystemMemoryReservation = systemMemory; } boolean fullyBlocked = hasRunningPipelines && runningPipelinesFullyBlocked; @@ -567,10 +553,8 @@ public TaskStats getTaskStats() blockedDrivers, completedDrivers, cumulativeUserMemory.get(), - cumulativeSystemMemory.get(), succinctBytes(userMemory), succinctBytes(taskMemoryContext.getRevocableMemory()), - succinctBytes(systemMemory), new Duration(totalScheduledTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalCpuTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalBlockedTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), diff --git a/core/trino-main/src/main/java/io/trino/operator/TaskStats.java b/core/trino-main/src/main/java/io/trino/operator/TaskStats.java index ea6658b09256..6373ea05a736 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TaskStats.java +++ b/core/trino-main/src/main/java/io/trino/operator/TaskStats.java @@ -52,10 +52,8 @@ public class TaskStats private final int completedDrivers; private final double cumulativeUserMemory; - private final double cumulativeSystemMemory; private final DataSize userMemoryReservation; private final DataSize revocableMemoryReservation; - private final DataSize systemMemoryReservation; private final Duration totalScheduledTime; private final Duration totalCpuTime; @@ -105,8 +103,6 @@ public TaskStats(DateTime createTime, DateTime endTime) 0, 0, 0.0, - 0.0, - DataSize.ofBytes(0), DataSize.ofBytes(0), DataSize.ofBytes(0), new Duration(0, MILLISECONDS), @@ -152,10 +148,8 @@ public TaskStats( @JsonProperty("completedDrivers") int completedDrivers, @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, - @JsonProperty("cumulativeSystemMemory") double cumulativeSystemMemory, @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation, - @JsonProperty("systemMemoryReservation") DataSize systemMemoryReservation, @JsonProperty("totalScheduledTime") Duration totalScheduledTime, @JsonProperty("totalCpuTime") Duration totalCpuTime, @@ -217,10 +211,8 @@ public TaskStats( this.completedDrivers = completedDrivers; this.cumulativeUserMemory = cumulativeUserMemory; - this.cumulativeSystemMemory = cumulativeSystemMemory; this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null"); this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null"); - this.systemMemoryReservation = requireNonNull(systemMemoryReservation, "systemMemoryReservation is null"); this.totalScheduledTime = requireNonNull(totalScheduledTime, "totalScheduledTime is null"); this.totalCpuTime = requireNonNull(totalCpuTime, "totalCpuTime is null"); @@ -340,12 +332,6 @@ public double getCumulativeUserMemory() return cumulativeUserMemory; } - @JsonProperty - public double getCumulativeSystemMemory() - { - return cumulativeSystemMemory; - } - @JsonProperty public DataSize getUserMemoryReservation() { @@ -358,12 +344,6 @@ public DataSize getRevocableMemoryReservation() return revocableMemoryReservation; } - @JsonProperty - public DataSize getSystemMemoryReservation() - { - return systemMemoryReservation; - } - @JsonProperty public Duration getTotalScheduledTime() { @@ -528,10 +508,8 @@ public TaskStats summarize() blockedDrivers, completedDrivers, cumulativeUserMemory, - cumulativeSystemMemory, userMemoryReservation, revocableMemoryReservation, - systemMemoryReservation, totalScheduledTime, totalCpuTime, totalBlockedTime, @@ -574,10 +552,8 @@ public TaskStats summarizeFinal() blockedDrivers, completedDrivers, cumulativeUserMemory, - cumulativeSystemMemory, userMemoryReservation, revocableMemoryReservation, - systemMemoryReservation, totalScheduledTime, totalCpuTime, totalBlockedTime, diff --git a/core/trino-main/src/main/java/io/trino/operator/TopNRankingOperator.java b/core/trino-main/src/main/java/io/trino/operator/TopNRankingOperator.java index 31c2b4d37f2a..b39b624cd4b4 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TopNRankingOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/TopNRankingOperator.java @@ -196,8 +196,7 @@ public TopNRankingOperator( { requireNonNull(maxPartialMemory, "maxPartialMemory is null"); this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); - // when the operator is final or maxPartialMemory is not set, use user memory instead of system memory - this.localMemoryContext = maxPartialMemory.isPresent() ? operatorContext.localSystemMemoryContext() : operatorContext.localUserMemoryContext(); + this.localMemoryContext = operatorContext.localUserMemoryContext(); ImmutableList.Builder outputChannelsBuilder = ImmutableList.builder(); for (int channel : requireNonNull(outputChannels, "outputChannels is null")) { diff --git a/core/trino-main/src/main/java/io/trino/operator/WindowOperator.java b/core/trino-main/src/main/java/io/trino/operator/WindowOperator.java index 2f7f1fbb4da8..7bc32aa40591 100644 --- a/core/trino-main/src/main/java/io/trino/operator/WindowOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/WindowOperator.java @@ -790,7 +790,7 @@ ListenableFuture spill() spiller = Optional.of(spillerFactory.create( sourceTypes, operatorContext.getSpillContext(), - operatorContext.newAggregateSystemMemoryContext())); + operatorContext.newAggregateUserMemoryContext())); } verify(inMemoryPagesIndexWithHashStrategies.pagesIndex.getPositionCount() > 0); diff --git a/core/trino-main/src/main/java/io/trino/operator/WorkProcessorOperatorAdapter.java b/core/trino-main/src/main/java/io/trino/operator/WorkProcessorOperatorAdapter.java index 763442204625..bc13b4cc693a 100644 --- a/core/trino-main/src/main/java/io/trino/operator/WorkProcessorOperatorAdapter.java +++ b/core/trino-main/src/main/java/io/trino/operator/WorkProcessorOperatorAdapter.java @@ -145,8 +145,7 @@ public WorkProcessorOperatorAdapter(OperatorContext operatorContext, AdapterWork this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); MemoryTrackingContext memoryTrackingContext = new MemoryTrackingContext( operatorContext.aggregateUserMemoryContext(), - operatorContext.aggregateRevocableMemoryContext(), - operatorContext.aggregateSystemMemoryContext()); + operatorContext.aggregateRevocableMemoryContext()); memoryTrackingContext.initializeLocalMemoryContexts(workProcessorOperatorFactory.getOperatorType()); this.workProcessorOperator = requireNonNull(workProcessorOperatorFactory, "workProcessorOperatorFactory is null") .createAdapterOperator(new ProcessorContext(operatorContext.getSession(), memoryTrackingContext, operatorContext)); diff --git a/core/trino-main/src/main/java/io/trino/operator/WorkProcessorPipelineSourceOperator.java b/core/trino-main/src/main/java/io/trino/operator/WorkProcessorPipelineSourceOperator.java index aca92e5422ee..7de696f8bc3c 100644 --- a/core/trino-main/src/main/java/io/trino/operator/WorkProcessorPipelineSourceOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/WorkProcessorPipelineSourceOperator.java @@ -298,8 +298,7 @@ private MemoryTrackingContext createMemoryTrackingContext(OperatorContext operat { return new MemoryTrackingContext( new InternalAggregatedMemoryContext(operatorContext.newAggregateUserMemoryContext(), () -> updatePeakMemoryReservations(operatorIndex)), - new InternalAggregatedMemoryContext(operatorContext.newAggregateRevocableMemoryContext(), () -> updatePeakMemoryReservations(operatorIndex)), - new InternalAggregatedMemoryContext(operatorContext.newAggregateSystemMemoryContext(), () -> updatePeakMemoryReservations(operatorIndex))); + new InternalAggregatedMemoryContext(operatorContext.newAggregateRevocableMemoryContext(), () -> updatePeakMemoryReservations(operatorIndex))); } private void updatePeakMemoryReservations(int operatorIndex) @@ -357,9 +356,7 @@ private List getNestedOperatorStats() succinctBytes(context.memoryTrackingContext.getUserMemory()), succinctBytes(context.memoryTrackingContext.getRevocableMemory()), - succinctBytes(context.memoryTrackingContext.getSystemMemory()), succinctBytes(context.peakUserMemoryReservation.get()), - succinctBytes(context.peakSystemMemoryReservation.get()), succinctBytes(context.peakRevocableMemoryReservation.get()), succinctBytes(context.peakTotalMemoryReservation.get()), DataSize.ofBytes(0), @@ -694,7 +691,6 @@ private static class WorkProcessorOperatorContext final AtomicReference connectorMetrics = new AtomicReference<>(Metrics.EMPTY); final AtomicLong peakUserMemoryReservation = new AtomicLong(); - final AtomicLong peakSystemMemoryReservation = new AtomicLong(); final AtomicLong peakRevocableMemoryReservation = new AtomicLong(); final AtomicLong peakTotalMemoryReservation = new AtomicLong(); @@ -720,11 +716,9 @@ private WorkProcessorOperatorContext( void updatePeakMemoryReservations() { long userMemory = memoryTrackingContext.getUserMemory(); - long systemMemory = memoryTrackingContext.getSystemMemory(); long revocableMemory = memoryTrackingContext.getRevocableMemory(); - long totalMemory = userMemory + systemMemory; + long totalMemory = userMemory; peakUserMemoryReservation.accumulateAndGet(userMemory, Math::max); - peakSystemMemoryReservation.accumulateAndGet(systemMemory, Math::max); peakRevocableMemoryReservation.accumulateAndGet(revocableMemory, Math::max); peakTotalMemoryReservation.accumulateAndGet(totalMemory, Math::max); } diff --git a/core/trino-main/src/main/java/io/trino/operator/WorkProcessorSourceOperatorAdapter.java b/core/trino-main/src/main/java/io/trino/operator/WorkProcessorSourceOperatorAdapter.java index b70cc6d27562..d3bb1ad02f85 100644 --- a/core/trino-main/src/main/java/io/trino/operator/WorkProcessorSourceOperatorAdapter.java +++ b/core/trino-main/src/main/java/io/trino/operator/WorkProcessorSourceOperatorAdapter.java @@ -78,8 +78,7 @@ public WorkProcessorSourceOperatorAdapter(OperatorContext operatorContext, Adapt operatorContext.getSession(), new MemoryTrackingContext( operatorContext.aggregateUserMemoryContext(), - operatorContext.aggregateRevocableMemoryContext(), - operatorContext.aggregateSystemMemoryContext()), + operatorContext.aggregateRevocableMemoryContext()), operatorContext.getDriverContext().getYieldSignal(), WorkProcessor.create(splitBuffer)); this.pages = sourceOperator.getOutputPages() diff --git a/core/trino-main/src/main/java/io/trino/operator/aggregation/builder/SpillableHashAggregationBuilder.java b/core/trino-main/src/main/java/io/trino/operator/aggregation/builder/SpillableHashAggregationBuilder.java index b391ea43e4b7..da7ad0f8461a 100644 --- a/core/trino-main/src/main/java/io/trino/operator/aggregation/builder/SpillableHashAggregationBuilder.java +++ b/core/trino-main/src/main/java/io/trino/operator/aggregation/builder/SpillableHashAggregationBuilder.java @@ -253,7 +253,7 @@ private ListenableFuture spillToDisk() spiller = Optional.of(spillerFactory.create( hashAggregationBuilder.buildTypes(), operatorContext.getSpillContext(), - operatorContext.newAggregateSystemMemoryContext())); + operatorContext.newAggregateUserMemoryContext())); } // start spilling process with current content of the hashAggregationBuilder builder... @@ -270,7 +270,7 @@ private WorkProcessor mergeFromDiskAndMemory() checkState(spiller.isPresent()); hashAggregationBuilder.setSpillOutput(); - mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext(), blockTypeOperators)); + mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateUserMemoryContext(), blockTypeOperators)); WorkProcessor mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, @@ -290,7 +290,7 @@ private WorkProcessor mergeFromDisk() { checkState(spiller.isPresent()); - mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext(), blockTypeOperators)); + mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateUserMemoryContext(), blockTypeOperators)); WorkProcessor mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, @@ -313,7 +313,7 @@ private WorkProcessor mergeSortedPages(WorkProcessor sortedPages, lo hashChannel, operatorContext, sortedPages, - operatorContext.aggregateSystemMemoryContext(), + operatorContext.aggregateUserMemoryContext(), memoryLimitForMerge, hashAggregationBuilder.getKeyChannels(), joinCompiler, diff --git a/core/trino-main/src/main/java/io/trino/operator/join/DefaultPageJoiner.java b/core/trino-main/src/main/java/io/trino/operator/join/DefaultPageJoiner.java index b7f86cef46e9..f9f5508dc63d 100644 --- a/core/trino-main/src/main/java/io/trino/operator/join/DefaultPageJoiner.java +++ b/core/trino-main/src/main/java/io/trino/operator/join/DefaultPageJoiner.java @@ -371,7 +371,7 @@ private Page spillAndMaskSpilledPositions(Page page, SpillInfoSnapshot spillInfo probeTypes, partitionGenerator.get(), spillContext.newLocalSpillContext(), - memoryTrackingContext.newAggregateSystemMemoryContext())); + memoryTrackingContext.newAggregateUserMemoryContext())); } PartitioningSpiller.PartitioningSpillResult result = spiller.get().partitionAndSpill(page, spillInfoSnapshot.getSpillMask()); diff --git a/core/trino-main/src/main/java/io/trino/operator/join/HashBuilderOperator.java b/core/trino-main/src/main/java/io/trino/operator/join/HashBuilderOperator.java index 7bfa135cd996..d0bbf8cc99da 100644 --- a/core/trino-main/src/main/java/io/trino/operator/join/HashBuilderOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/join/HashBuilderOperator.java @@ -433,7 +433,7 @@ private ListenableFuture spillIndex() spiller = Optional.of(singleStreamSpillerFactory.create( index.getTypes(), operatorContext.getSpillContext().newLocalSpillContext(), - operatorContext.newLocalSystemMemoryContext(HashBuilderOperator.class.getSimpleName()))); + operatorContext.newLocalUserMemoryContext(HashBuilderOperator.class.getSimpleName()))); return getSpiller().spill(index.getPages()); } diff --git a/core/trino-main/src/main/java/io/trino/operator/output/PartitionedOutputOperator.java b/core/trino-main/src/main/java/io/trino/operator/output/PartitionedOutputOperator.java index d1eb6155f4cb..43f5d4ddc649 100644 --- a/core/trino-main/src/main/java/io/trino/operator/output/PartitionedOutputOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/output/PartitionedOutputOperator.java @@ -214,7 +214,7 @@ public OperatorFactory duplicate() private final OperatorContext operatorContext; private final Function pagePreprocessor; private final PagePartitioner partitionFunction; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; private final long partitionsInitialRetainedSize; private ListenableFuture isBlocked = NOT_BLOCKED; private boolean finished; @@ -248,9 +248,9 @@ public PartitionedOutputOperator( operatorContext); operatorContext.setInfoSupplier(this.partitionFunction.getOperatorInfoSupplier()); - this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(PartitionedOutputOperator.class.getSimpleName()); + this.memoryContext = operatorContext.newLocalUserMemoryContext(PartitionedOutputOperator.class.getSimpleName()); this.partitionsInitialRetainedSize = this.partitionFunction.getRetainedSizeInBytes(); - this.systemMemoryContext.setBytes(partitionsInitialRetainedSize); + this.memoryContext.setBytes(partitionsInitialRetainedSize); } @Override @@ -309,7 +309,7 @@ public void addInput(Page page) long partitionsSizeInBytes = partitionFunction.getSizeInBytes(); // We also add partitionsInitialRetainedSize as an approximation of the object overhead of the partitions. - systemMemoryContext.setBytes(partitionsSizeInBytes + partitionsInitialRetainedSize); + memoryContext.setBytes(partitionsSizeInBytes + partitionsInitialRetainedSize); } @Override @@ -321,7 +321,7 @@ public Page getOutput() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } public static class PartitionedOutputInfo diff --git a/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java b/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java index b676b98b464d..744ba652afe2 100644 --- a/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java +++ b/core/trino-main/src/main/java/io/trino/server/BasicQueryStats.java @@ -55,7 +55,6 @@ public class BasicQueryStats private final DataSize physicalInputDataSize; private final double cumulativeUserMemory; - private final double cumulativeSystemMemory; private final DataSize userMemoryReservation; private final DataSize totalMemoryReservation; private final DataSize peakUserMemoryReservation; @@ -83,7 +82,6 @@ public BasicQueryStats( @JsonProperty("rawInputPositions") long rawInputPositions, @JsonProperty("physicalInputDataSize") DataSize physicalInputDataSize, @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, - @JsonProperty("cumulativeSystemMemory") double cumulativeSystemMemory, @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation, @JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation, @@ -115,7 +113,6 @@ public BasicQueryStats( this.physicalInputDataSize = physicalInputDataSize; this.cumulativeUserMemory = cumulativeUserMemory; - this.cumulativeSystemMemory = cumulativeSystemMemory; this.userMemoryReservation = userMemoryReservation; this.totalMemoryReservation = totalMemoryReservation; this.peakUserMemoryReservation = peakUserMemoryReservation; @@ -144,7 +141,6 @@ public BasicQueryStats(QueryStats queryStats) queryStats.getRawInputPositions(), queryStats.getPhysicalInputDataSize(), queryStats.getCumulativeUserMemory(), - queryStats.getCumulativeSystemMemory(), queryStats.getUserMemoryReservation(), queryStats.getTotalMemoryReservation(), queryStats.getPeakUserMemoryReservation(), @@ -173,7 +169,6 @@ public static BasicQueryStats immediateFailureQueryStats() 0, DataSize.ofBytes(0), 0, - 0, DataSize.ofBytes(0), DataSize.ofBytes(0), DataSize.ofBytes(0), @@ -263,12 +258,6 @@ public double getCumulativeUserMemory() return cumulativeUserMemory; } - @JsonProperty - public double getCumulativeSystemMemory() - { - return cumulativeSystemMemory; - } - @JsonProperty public DataSize getUserMemoryReservation() { diff --git a/core/trino-main/src/main/java/io/trino/split/MappedPageSource.java b/core/trino-main/src/main/java/io/trino/split/MappedPageSource.java index 56f2e352521e..1d021b8687d5 100644 --- a/core/trino-main/src/main/java/io/trino/split/MappedPageSource.java +++ b/core/trino-main/src/main/java/io/trino/split/MappedPageSource.java @@ -63,9 +63,9 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return delegate.getSystemMemoryUsage(); + return delegate.getMemoryUsage(); } @Override diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java index 5c0fa494e264..9ffb9754f566 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java @@ -3130,8 +3130,7 @@ public PhysicalOperation visitTableWriter(TableWriterNode node, LocalExecutionPl STATS_START_CHANNEL, outputMapping, source, - context, - true); + context); } return createHashAggregationOperatorFactory( node.getId(), @@ -3155,8 +3154,7 @@ public PhysicalOperation visitTableWriter(TableWriterNode node, LocalExecutionPl // is not possible, as it doesn't accept raw input data. // Disabling partial pre-aggregation memory limit effectively // turns PARTIAL aggregation into INTERMEDIATE. - Optional.empty(), - true); + Optional.empty()); }).orElse(new DevNullOperatorFactory(context.getNextOperatorId(), node.getId())); List inputChannels = node.getColumns().stream() @@ -3214,8 +3212,7 @@ public PhysicalOperation visitTableFinish(TableFinishNode node, LocalExecutionPl 0, outputMapping, source, - context, - true); + context); } return createHashAggregationOperatorFactory( node.getId(), @@ -3235,8 +3232,7 @@ public PhysicalOperation visitTableFinish(TableFinishNode node, LocalExecutionPl outputMapping, 200, // final aggregation ignores partial pre-aggregation memory limit - Optional.empty(), - true); + Optional.empty()); }).orElse(new DevNullOperatorFactory(context.getNextOperatorId(), node.getId())); Map aggregationOutput = outputMapping.buildOrThrow(); @@ -3689,8 +3685,7 @@ private PhysicalOperation planGlobalAggregation(AggregationNode node, PhysicalOp 0, outputMappings, source, - context, - node.getStep().isOutputPartial()); + context); return new PhysicalOperation(operatorFactory, outputMappings.buildOrThrow(), context, source); } @@ -3701,8 +3696,7 @@ private AggregationOperatorFactory createAggregationOperatorFactory( int startOutputChannel, ImmutableMap.Builder outputMappings, PhysicalOperation source, - LocalExecutionPlanContext context, - boolean useSystemMemory) + LocalExecutionPlanContext context) { int outputChannel = startOutputChannel; ImmutableList.Builder aggregatorFactories = ImmutableList.builder(); @@ -3713,7 +3707,7 @@ private AggregationOperatorFactory createAggregationOperatorFactory( outputMappings.put(symbol, outputChannel); // one aggregation per channel outputChannel++; } - return new AggregationOperatorFactory(context.getNextOperatorId(), planNodeId, aggregatorFactories.build(), useSystemMemory); + return new AggregationOperatorFactory(context.getNextOperatorId(), planNodeId, aggregatorFactories.build()); } private PhysicalOperation planGroupByAggregation( @@ -3741,8 +3735,7 @@ private PhysicalOperation planGroupByAggregation( 0, mappings, 10_000, - Optional.of(maxPartialAggregationMemorySize), - node.getStep().isOutputPartial()); + Optional.of(maxPartialAggregationMemorySize)); return new PhysicalOperation(operatorFactory, mappings.buildOrThrow(), context, source); } @@ -3763,8 +3756,7 @@ private OperatorFactory createHashAggregationOperatorFactory( int startOutputChannel, ImmutableMap.Builder outputMappings, int expectedGroups, - Optional maxPartialAggregationMemorySize, - boolean useSystemMemory) + Optional maxPartialAggregationMemorySize) { List aggregationOutputSymbols = new ArrayList<>(); List aggregatorFactories = new ArrayList<>(); @@ -3810,7 +3802,6 @@ private OperatorFactory createHashAggregationOperatorFactory( source.getTypes(), groupByTypes, groupByChannels, - step, aggregatorFactories, joinCompiler); } @@ -3833,8 +3824,7 @@ private OperatorFactory createHashAggregationOperatorFactory( unspillMemoryLimit, spillerFactory, joinCompiler, - blockTypeOperators, - useSystemMemory); + blockTypeOperators); } } } diff --git a/core/trino-main/src/main/java/io/trino/testing/TestingTaskContext.java b/core/trino-main/src/main/java/io/trino/testing/TestingTaskContext.java index 516197f3a3e8..2fb813563961 100644 --- a/core/trino-main/src/main/java/io/trino/testing/TestingTaskContext.java +++ b/core/trino-main/src/main/java/io/trino/testing/TestingTaskContext.java @@ -88,7 +88,6 @@ public static class Builder private QueryId queryId = new QueryId("test_query"); private TaskStateMachine taskStateMachine; private DataSize queryMaxMemory = DataSize.of(256, MEGABYTE); - private final DataSize queryMaxTotalMemory = DataSize.of(512, MEGABYTE); private DataSize memoryPoolSize = DataSize.of(1, GIGABYTE); private DataSize maxSpillSize = DataSize.of(1, GIGABYTE); private DataSize queryMaxSpillSize = DataSize.of(1, GIGABYTE); @@ -144,9 +143,9 @@ public TaskContext build() QueryContext queryContext = new QueryContext( queryId, queryMaxMemory, - queryMaxTotalMemory, Optional.empty(), memoryPool, + 0L, GC_MONITOR, notificationExecutor, yieldExecutor, diff --git a/core/trino-main/src/main/resources/webapp/dist/query.js b/core/trino-main/src/main/resources/webapp/dist/query.js index e2c6b801fb4e..b7ba760701b0 100644 --- a/core/trino-main/src/main/resources/webapp/dist/query.js +++ b/core/trino-main/src/main/resources/webapp/dist/query.js @@ -106,7 +106,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.QueryDetail = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _reactable = __webpack_require__(/*! reactable */ \"./node_modules/reactable/lib/reactable.js\");\n\nvar _reactable2 = _interopRequireDefault(_reactable);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nvar Table = _reactable2.default.Table,\n Thead = _reactable2.default.Thead,\n Th = _reactable2.default.Th,\n Tr = _reactable2.default.Tr,\n Td = _reactable2.default.Td;\n\nvar TaskList = function (_React$Component) {\n _inherits(TaskList, _React$Component);\n\n function TaskList() {\n _classCallCheck(this, TaskList);\n\n return _possibleConstructorReturn(this, (TaskList.__proto__ || Object.getPrototypeOf(TaskList)).apply(this, arguments));\n }\n\n _createClass(TaskList, [{\n key: \"render\",\n value: function render() {\n var tasks = this.props.tasks;\n\n if (tasks === undefined || tasks.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"No threads in the selected group\"\n )\n )\n );\n }\n\n var showPortNumbers = TaskList.showPortNumbers(tasks);\n\n var renderedTasks = tasks.map(function (task) {\n var elapsedTime = (0, _utils.parseDuration)(task.stats.elapsedTime);\n if (elapsedTime === 0) {\n elapsedTime = Date.now() - Date.parse(task.stats.createTime);\n }\n\n return _react2.default.createElement(\n Tr,\n { key: task.taskStatus.taskId },\n _react2.default.createElement(\n Td,\n { column: \"id\", value: task.taskStatus.taskId },\n _react2.default.createElement(\n \"a\",\n { href: \"/ui/api/worker/\" + task.taskStatus.nodeId + \"/task/\" + task.taskStatus.taskId + \"?pretty\" },\n (0, _utils.getTaskIdSuffix)(task.taskStatus.taskId)\n )\n ),\n _react2.default.createElement(\n Td,\n { column: \"host\", value: (0, _utils.getHostname)(task.taskStatus.self) },\n _react2.default.createElement(\n \"a\",\n { href: \"worker.html?\" + task.taskStatus.nodeId, className: \"font-light\", target: \"_blank\" },\n showPortNumbers ? (0, _utils.getHostAndPort)(task.taskStatus.self) : (0, _utils.getHostname)(task.taskStatus.self)\n )\n ),\n _react2.default.createElement(\n Td,\n { column: \"state\", value: TaskList.formatState(task.taskStatus.state, task.stats.fullyBlocked) },\n TaskList.formatState(task.taskStatus.state, task.stats.fullyBlocked)\n ),\n _react2.default.createElement(\n Td,\n { column: \"rows\", value: task.stats.rawInputPositions },\n (0, _utils.formatCount)(task.stats.rawInputPositions)\n ),\n _react2.default.createElement(\n Td,\n { column: \"rowsSec\", value: (0, _utils.computeRate)(task.stats.rawInputPositions, elapsedTime) },\n (0, _utils.formatCount)((0, _utils.computeRate)(task.stats.rawInputPositions, elapsedTime))\n ),\n _react2.default.createElement(\n Td,\n { column: \"bytes\", value: (0, _utils.parseDataSize)(task.stats.rawInputDataSize) },\n (0, _utils.formatDataSizeBytes)((0, _utils.parseDataSize)(task.stats.rawInputDataSize))\n ),\n _react2.default.createElement(\n Td,\n { column: \"bytesSec\", value: (0, _utils.computeRate)((0, _utils.parseDataSize)(task.stats.rawInputDataSize), elapsedTime) },\n (0, _utils.formatDataSizeBytes)((0, _utils.computeRate)((0, _utils.parseDataSize)(task.stats.rawInputDataSize), elapsedTime))\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsPending\", value: task.stats.queuedDrivers },\n task.stats.queuedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsRunning\", value: task.stats.runningDrivers },\n task.stats.runningDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsBlocked\", value: task.stats.blockedDrivers },\n task.stats.blockedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsDone\", value: task.stats.completedDrivers },\n task.stats.completedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"elapsedTime\", value: (0, _utils.parseDuration)(task.stats.elapsedTime) },\n task.stats.elapsedTime\n ),\n _react2.default.createElement(\n Td,\n { column: \"cpuTime\", value: (0, _utils.parseDuration)(task.stats.totalCpuTime) },\n task.stats.totalCpuTime\n ),\n _react2.default.createElement(\n Td,\n { column: \"bufferedBytes\", value: task.outputBuffers.totalBufferedBytes },\n (0, _utils.formatDataSizeBytes)(task.outputBuffers.totalBufferedBytes)\n )\n );\n });\n\n return _react2.default.createElement(\n Table,\n { id: \"tasks\", className: \"table table-striped sortable\", sortable: [{\n column: 'id',\n sortFunction: TaskList.compareTaskId\n }, 'host', 'state', 'splitsPending', 'splitsRunning', 'splitsBlocked', 'splitsDone', 'rows', 'rowsSec', 'bytes', 'bytesSec', 'elapsedTime', 'cpuTime', 'bufferedBytes'],\n defaultSort: { column: 'id', direction: 'asc' } },\n _react2.default.createElement(\n Thead,\n null,\n _react2.default.createElement(\n Th,\n { column: \"id\" },\n \"ID\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"host\" },\n \"Host\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"state\" },\n \"State\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsPending\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-pause\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Pending splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsRunning\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-play\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Running splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsBlocked\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-bookmark\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Blocked splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsDone\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-ok\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Completed splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"rows\" },\n \"Rows\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"rowsSec\" },\n \"Rows/s\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bytes\" },\n \"Bytes\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bytesSec\" },\n \"Bytes/s\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"elapsedTime\" },\n \"Elapsed\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"cpuTime\" },\n \"CPU Time\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bufferedBytes\" },\n \"Buffered\"\n )\n ),\n renderedTasks\n );\n }\n }], [{\n key: \"removeQueryId\",\n value: function removeQueryId(id) {\n var pos = id.indexOf('.');\n if (pos !== -1) {\n return id.substring(pos + 1);\n }\n return id;\n }\n }, {\n key: \"compareTaskId\",\n value: function compareTaskId(taskA, taskB) {\n var taskIdArrA = TaskList.removeQueryId(taskA).split(\".\");\n var taskIdArrB = TaskList.removeQueryId(taskB).split(\".\");\n\n if (taskIdArrA.length > taskIdArrB.length) {\n return 1;\n }\n for (var i = 0; i < taskIdArrA.length; i++) {\n var anum = Number.parseInt(taskIdArrA[i]);\n var bnum = Number.parseInt(taskIdArrB[i]);\n if (anum !== bnum) {\n return anum > bnum ? 1 : -1;\n }\n }\n\n return 0;\n }\n }, {\n key: \"showPortNumbers\",\n value: function showPortNumbers(tasks) {\n // check if any host has multiple port numbers\n var hostToPortNumber = {};\n for (var i = 0; i < tasks.length; i++) {\n var taskUri = tasks[i].taskStatus.self;\n var hostname = (0, _utils.getHostname)(taskUri);\n var port = (0, _utils.getPort)(taskUri);\n if (hostname in hostToPortNumber && hostToPortNumber[hostname] !== port) {\n return true;\n }\n hostToPortNumber[hostname] = port;\n }\n\n return false;\n }\n }, {\n key: \"formatState\",\n value: function formatState(state, fullyBlocked) {\n if (fullyBlocked && state === \"RUNNING\") {\n return \"BLOCKED\";\n } else {\n return state;\n }\n }\n }]);\n\n return TaskList;\n}(_react2.default.Component);\n\nvar BAR_CHART_WIDTH = 800;\n\nvar BAR_CHART_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#8997B3',\n chartRangeMin: 0,\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: 'Task {{offset:offset}} - {{value}}',\n disableHiddenCheck: true\n};\n\nvar HISTOGRAM_WIDTH = 175;\n\nvar HISTOGRAM_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#747F96',\n zeroAxis: true,\n chartRangeMin: 0,\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: '{{offset:offset}} -- {{value}} tasks',\n disableHiddenCheck: true\n};\n\nvar StageSummary = function (_React$Component2) {\n _inherits(StageSummary, _React$Component2);\n\n function StageSummary(props) {\n _classCallCheck(this, StageSummary);\n\n var _this2 = _possibleConstructorReturn(this, (StageSummary.__proto__ || Object.getPrototypeOf(StageSummary)).call(this, props));\n\n _this2.state = {\n expanded: false,\n lastRender: null\n };\n return _this2;\n }\n\n _createClass(StageSummary, [{\n key: \"getExpandedIcon\",\n value: function getExpandedIcon() {\n return this.state.expanded ? \"glyphicon-chevron-up\" : \"glyphicon-chevron-down\";\n }\n }, {\n key: \"getExpandedStyle\",\n value: function getExpandedStyle() {\n return this.state.expanded ? {} : { display: \"none\" };\n }\n }, {\n key: \"toggleExpanded\",\n value: function toggleExpanded() {\n this.setState({\n expanded: !this.state.expanded\n });\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n var stage = this.props.stage;\n var numTasks = stage.tasks.length;\n\n // sort the x-axis\n stage.tasks.sort(function (taskA, taskB) {\n return (0, _utils.getTaskNumber)(taskA.taskStatus.taskId) - (0, _utils.getTaskNumber)(taskB.taskStatus.taskId);\n });\n\n var scheduledTimes = stage.tasks.map(function (task) {\n return (0, _utils.parseDuration)(task.stats.totalScheduledTime);\n });\n var cpuTimes = stage.tasks.map(function (task) {\n return (0, _utils.parseDuration)(task.stats.totalCpuTime);\n });\n\n // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts\n if (this.state.lastRender === null || Date.now() - this.state.lastRender >= 1000) {\n var renderTimestamp = Date.now();\n var stageId = (0, _utils.getStageNumber)(stage.stageId);\n\n StageSummary.renderHistogram('#scheduled-time-histogram-' + stageId, scheduledTimes, _utils.formatDuration);\n StageSummary.renderHistogram('#cpu-time-histogram-' + stageId, cpuTimes, _utils.formatDuration);\n\n if (this.state.expanded) {\n // this needs to be a string otherwise it will also be passed to numberFormatter\n var tooltipValueLookups = { 'offset': {} };\n for (var i = 0; i < numTasks; i++) {\n tooltipValueLookups['offset'][i] = (0, _utils.getStageNumber)(stage.stageId) + \".\" + i;\n }\n\n var stageBarChartProperties = $.extend({}, BAR_CHART_PROPERTIES, { barWidth: BAR_CHART_WIDTH / numTasks, tooltipValueLookups: tooltipValueLookups });\n\n $('#scheduled-time-bar-chart-' + stageId).sparkline(scheduledTimes, $.extend({}, stageBarChartProperties, { numberFormatter: _utils.formatDuration }));\n $('#cpu-time-bar-chart-' + stageId).sparkline(cpuTimes, $.extend({}, stageBarChartProperties, { numberFormatter: _utils.formatDuration }));\n }\n\n this.setState({\n lastRender: renderTimestamp\n });\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n if (stage === undefined || !stage.hasOwnProperty('plan')) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Information about this stage is unavailable.\"\n )\n );\n }\n\n var totalBufferedBytes = stage.tasks.map(function (task) {\n return task.outputBuffers.totalBufferedBytes;\n }).reduce(function (a, b) {\n return a + b;\n }, 0);\n\n var stageId = (0, _utils.getStageNumber)(stage.stageId);\n\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-id\" },\n _react2.default.createElement(\n \"div\",\n { className: \"stage-state-color\", style: { borderLeftColor: (0, _utils.getStageStateColor)(stage) } },\n stageId\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"table single-stage-table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-time\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Time\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Scheduled\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalScheduledTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalBlockedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"CPU\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalCpuTime\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-memory\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Memory\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Cumulative\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.formatDataSizeBytes)(stage.stageStats.cumulativeUserMemory / 1000)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Current\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.parseAndFormatDataSize)(stage.stageStats.userMemoryReservation)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Buffers\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.formatDataSize)(totalBufferedBytes)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Peak\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.parseAndFormatDataSize)(stage.stageStats.peakUserMemoryReservation)\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-tasks\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Tasks\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Pending\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.taskStatus.state === \"PLANNED\";\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Running\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.taskStatus.state === \"RUNNING\";\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.stats.fullyBlocked;\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Total\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.length\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table histogram-table\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-chart-header\" },\n \"Scheduled Time Skew\"\n )\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"histogram-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"histogram\", id: \"scheduled-time-histogram-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table histogram-table\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-chart-header\" },\n \"CPU Time Skew\"\n )\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"histogram-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"histogram\", id: \"cpu-time-histogram-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"expand-charts-container\" },\n _react2.default.createElement(\n \"a\",\n { onClick: this.toggleExpanded.bind(this), className: \"expand-charts-button\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon \" + this.getExpandedIcon(), style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\", title: \"More\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"expanded-chart\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title expanded-chart-title\" },\n \"Task Scheduled Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"bar-chart-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"bar-chart\", id: \"scheduled-time-bar-chart-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"expanded-chart\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title expanded-chart-title\" },\n \"Task CPU Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"bar-chart-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"bar-chart\", id: \"cpu-time-bar-chart-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n )\n )\n )\n )\n )\n );\n }\n }], [{\n key: \"renderHistogram\",\n value: function renderHistogram(histogramId, inputData, numberFormatter) {\n var numBuckets = Math.min(HISTOGRAM_WIDTH, Math.sqrt(inputData.length));\n var dataMin = Math.min.apply(null, inputData);\n var dataMax = Math.max.apply(null, inputData);\n var bucketSize = (dataMax - dataMin) / numBuckets;\n\n var histogramData = [];\n if (bucketSize === 0) {\n histogramData = [inputData.length];\n } else {\n for (var i = 0; i < numBuckets + 1; i++) {\n histogramData.push(0);\n }\n\n for (var _i in inputData) {\n var dataPoint = inputData[_i];\n var bucket = Math.floor((dataPoint - dataMin) / bucketSize);\n histogramData[bucket] = histogramData[bucket] + 1;\n }\n }\n\n var tooltipValueLookups = { 'offset': {} };\n for (var _i2 = 0; _i2 < histogramData.length; _i2++) {\n tooltipValueLookups['offset'][_i2] = numberFormatter(dataMin + _i2 * bucketSize) + \"-\" + numberFormatter(dataMin + (_i2 + 1) * bucketSize);\n }\n\n var stageHistogramProperties = $.extend({}, HISTOGRAM_PROPERTIES, { barWidth: HISTOGRAM_WIDTH / histogramData.length, tooltipValueLookups: tooltipValueLookups });\n $(histogramId).sparkline(histogramData, stageHistogramProperties);\n }\n }]);\n\n return StageSummary;\n}(_react2.default.Component);\n\nvar StageList = function (_React$Component3) {\n _inherits(StageList, _React$Component3);\n\n function StageList() {\n _classCallCheck(this, StageList);\n\n return _possibleConstructorReturn(this, (StageList.__proto__ || Object.getPrototypeOf(StageList)).apply(this, arguments));\n }\n\n _createClass(StageList, [{\n key: \"getStages\",\n value: function getStages(stage) {\n if (stage === undefined || !stage.hasOwnProperty('subStages')) {\n return [];\n }\n\n return [].concat.apply(stage, stage.subStages.map(this.getStages, this));\n }\n }, {\n key: \"render\",\n value: function render() {\n var stages = this.getStages(this.props.outputStage);\n\n if (stages === undefined || stages.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n \"No stage information available.\"\n )\n );\n }\n\n var renderedStages = stages.map(function (stage) {\n return _react2.default.createElement(StageSummary, { key: stage.stageId, stage: stage });\n });\n\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\", id: \"stage-list\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n renderedStages\n )\n )\n )\n );\n }\n }]);\n\n return StageList;\n}(_react2.default.Component);\n\nvar SMALL_SPARKLINE_PROPERTIES = {\n width: '100%',\n height: '57px',\n fillColor: '#3F4552',\n lineColor: '#747F96',\n spotColor: '#1EDCFF',\n tooltipClassname: 'sparkline-tooltip',\n disableHiddenCheck: true\n};\n\nvar TASK_FILTER = {\n ALL: function ALL() {\n return true;\n },\n PLANNED: function PLANNED(state) {\n return state === 'PLANNED';\n },\n RUNNING: function RUNNING(state) {\n return state === 'RUNNING';\n },\n FINISHED: function FINISHED(state) {\n return state === 'FINISHED';\n },\n FAILED: function FAILED(state) {\n return state === 'FAILED' || state === 'ABORTED' || state === 'CANCELED';\n }\n};\n\nvar QueryDetail = exports.QueryDetail = function (_React$Component4) {\n _inherits(QueryDetail, _React$Component4);\n\n function QueryDetail(props) {\n _classCallCheck(this, QueryDetail);\n\n var _this4 = _possibleConstructorReturn(this, (QueryDetail.__proto__ || Object.getPrototypeOf(QueryDetail)).call(this, props));\n\n _this4.state = {\n query: null,\n lastSnapshotStages: null,\n lastSnapshotTasks: null,\n\n lastScheduledTime: 0,\n lastCpuTime: 0,\n lastRowInput: 0,\n lastByteInput: 0,\n lastPhysicalInput: 0,\n lastPhysicalTime: 0,\n\n scheduledTimeRate: [],\n cpuTimeRate: [],\n rowInputRate: [],\n byteInputRate: [],\n physicalInputRate: [],\n\n reservedMemory: [],\n\n initialized: false,\n queryEnded: false,\n renderingEnded: false,\n\n lastRefresh: null,\n lastRender: null,\n\n stageRefresh: true,\n taskRefresh: true,\n\n taskFilter: TASK_FILTER.ALL\n };\n\n _this4.refreshLoop = _this4.refreshLoop.bind(_this4);\n return _this4;\n }\n\n _createClass(QueryDetail, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.queryEnded) {\n // task.info-update-interval is set to 3 seconds by default\n this.timeoutId = setTimeout(this.refreshLoop, 3000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this5 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n var queryId = (0, _utils.getFirstParameter)(window.location.search);\n $.get('/ui/api/query/' + queryId, function (query) {\n var lastSnapshotStages = this.state.lastSnapshotStage;\n if (this.state.stageRefresh) {\n lastSnapshotStages = query.outputStage;\n }\n var lastSnapshotTasks = this.state.lastSnapshotTasks;\n if (this.state.taskRefresh) {\n lastSnapshotTasks = query.outputStage;\n }\n\n var lastRefresh = this.state.lastRefresh;\n var lastScheduledTime = this.state.lastScheduledTime;\n var lastCpuTime = this.state.lastCpuTime;\n var lastRowInput = this.state.lastRowInput;\n var lastByteInput = this.state.lastByteInput;\n var lastPhysicalInput = this.state.lastPhysicalInput;\n var lastPhysicalTime = this.state.lastPhysicalTime;\n var alreadyEnded = this.state.queryEnded;\n var nowMillis = Date.now();\n\n this.setState({\n query: query,\n lastSnapshotStage: lastSnapshotStages,\n lastSnapshotTasks: lastSnapshotTasks,\n\n lastPhysicalTime: (0, _utils.parseDuration)(query.queryStats.physicalInputReadTime),\n lastScheduledTime: (0, _utils.parseDuration)(query.queryStats.totalScheduledTime),\n lastCpuTime: (0, _utils.parseDuration)(query.queryStats.totalCpuTime),\n lastRowInput: query.queryStats.processedInputPositions,\n lastByteInput: (0, _utils.parseDataSize)(query.queryStats.processedInputDataSize),\n lastPhysicalInput: (0, _utils.parseDataSize)(query.queryStats.physicalInputDataSize),\n\n initialized: true,\n queryEnded: !!query.finalQueryInfo,\n\n lastRefresh: nowMillis\n });\n\n // i.e. don't show sparklines if we've already decided not to update or if we don't have one previous measurement\n if (alreadyEnded || lastRefresh === null && query.state === \"RUNNING\") {\n this.resetTimer();\n return;\n }\n\n if (lastRefresh === null) {\n lastRefresh = nowMillis - (0, _utils.parseDuration)(query.queryStats.elapsedTime);\n }\n\n var elapsedSecsSinceLastRefresh = (nowMillis - lastRefresh) / 1000.0;\n if (elapsedSecsSinceLastRefresh >= 0) {\n var currentScheduledTimeRate = ((0, _utils.parseDuration)(query.queryStats.totalScheduledTime) - lastScheduledTime) / (elapsedSecsSinceLastRefresh * 1000);\n var currentCpuTimeRate = ((0, _utils.parseDuration)(query.queryStats.totalCpuTime) - lastCpuTime) / (elapsedSecsSinceLastRefresh * 1000);\n var currentPhysicalReadTime = ((0, _utils.parseDuration)(query.queryStats.physicalInputReadTime) - lastPhysicalTime) / 1000;\n var currentRowInputRate = (query.queryStats.processedInputPositions - lastRowInput) / elapsedSecsSinceLastRefresh;\n var currentByteInputRate = ((0, _utils.parseDataSize)(query.queryStats.processedInputDataSize) - lastByteInput) / elapsedSecsSinceLastRefresh;\n var currentPhysicalInputRate = currentPhysicalReadTime > 0 ? ((0, _utils.parseDataSize)(query.queryStats.physicalInputDataSize) - lastPhysicalInput) / currentPhysicalReadTime : 0;\n\n this.setState({\n scheduledTimeRate: (0, _utils.addToHistory)(currentScheduledTimeRate, this.state.scheduledTimeRate),\n cpuTimeRate: (0, _utils.addToHistory)(currentCpuTimeRate, this.state.cpuTimeRate),\n rowInputRate: (0, _utils.addToHistory)(currentRowInputRate, this.state.rowInputRate),\n byteInputRate: (0, _utils.addToHistory)(currentByteInputRate, this.state.byteInputRate),\n reservedMemory: (0, _utils.addToHistory)((0, _utils.parseDataSize)(query.queryStats.totalMemoryReservation), this.state.reservedMemory),\n physicalInputRate: (0, _utils.addToHistory)(currentPhysicalInputRate, this.state.physicalInputRate)\n });\n }\n this.resetTimer();\n }.bind(this)).fail(function () {\n _this5.setState({\n initialized: true\n });\n _this5.resetTimer();\n });\n }\n }, {\n key: \"handleTaskRefreshClick\",\n value: function handleTaskRefreshClick() {\n if (this.state.taskRefresh) {\n this.setState({\n taskRefresh: false,\n lastSnapshotTasks: this.state.query.outputStage\n });\n } else {\n this.setState({\n taskRefresh: true\n });\n }\n }\n }, {\n key: \"renderTaskRefreshButton\",\n value: function renderTaskRefreshButton() {\n if (this.state.taskRefresh) {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleTaskRefreshClick.bind(this) },\n \"Auto-Refresh: On\"\n );\n } else {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleTaskRefreshClick.bind(this) },\n \"Auto-Refresh: Off\"\n );\n }\n }\n }, {\n key: \"handleStageRefreshClick\",\n value: function handleStageRefreshClick() {\n if (this.state.stageRefresh) {\n this.setState({\n stageRefresh: false,\n lastSnapshotStages: this.state.query.outputStage\n });\n } else {\n this.setState({\n stageRefresh: true\n });\n }\n }\n }, {\n key: \"renderStageRefreshButton\",\n value: function renderStageRefreshButton() {\n if (this.state.stageRefresh) {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleStageRefreshClick.bind(this) },\n \"Auto-Refresh: On\"\n );\n } else {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleStageRefreshClick.bind(this) },\n \"Auto-Refresh: Off\"\n );\n }\n }\n }, {\n key: \"renderTaskFilterListItem\",\n value: function renderTaskFilterListItem(taskFilter, taskFilterText) {\n return _react2.default.createElement(\n \"li\",\n null,\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: this.state.taskFilter === taskFilter ? \"selected\" : \"\", onClick: this.handleTaskFilterClick.bind(this, taskFilter) },\n taskFilterText\n )\n );\n }\n }, {\n key: \"handleTaskFilterClick\",\n value: function handleTaskFilterClick(filter, event) {\n this.setState({\n taskFilter: filter\n });\n event.preventDefault();\n }\n }, {\n key: \"getTasksFromStage\",\n value: function getTasksFromStage(stage) {\n if (stage === undefined || !stage.hasOwnProperty('subStages') || !stage.hasOwnProperty('tasks')) {\n return [];\n }\n\n return [].concat.apply(stage.tasks, stage.subStages.map(this.getTasksFromStage, this));\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop();\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts\n if (this.state.lastRender === null || Date.now() - this.state.lastRender >= 1000 || this.state.ended && !this.state.renderingEnded) {\n var renderTimestamp = Date.now();\n $('#scheduled-time-rate-sparkline').sparkline(this.state.scheduledTimeRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, {\n chartRangeMin: 0,\n numberFormatter: _utils.precisionRound\n }));\n $('#cpu-time-rate-sparkline').sparkline(this.state.cpuTimeRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#row-input-rate-sparkline').sparkline(this.state.rowInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatCount }));\n $('#byte-input-rate-sparkline').sparkline(this.state.byteInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n $('#reserved-memory-sparkline').sparkline(this.state.reservedMemory, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n $('#physical-input-rate-sparkline').sparkline(this.state.physicalInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n\n if (this.state.lastRender === null) {\n $('#query').each(function (i, block) {\n hljs.highlightBlock(block);\n });\n\n $('#prepared-query').each(function (i, block) {\n hljs.highlightBlock(block);\n });\n }\n\n this.setState({\n renderingEnded: this.state.ended,\n lastRender: renderTimestamp\n });\n }\n\n $('[data-toggle=\"tooltip\"]').tooltip();\n new window.ClipboardJS('.copy-button');\n }\n }, {\n key: \"renderTasks\",\n value: function renderTasks() {\n var _this6 = this;\n\n if (this.state.lastSnapshotTasks === null) {\n return;\n }\n\n var tasks = this.getTasksFromStage(this.state.lastSnapshotTasks).filter(function (task) {\n return _this6.state.taskFilter(task.taskStatus.state);\n }, this);\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-9\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Tasks\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-3\" },\n _react2.default.createElement(\n \"table\",\n { className: \"header-inline-links\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"input-group-btn text-right\" },\n _react2.default.createElement(\n \"button\",\n { type: \"button\", className: \"btn btn-default dropdown-toggle pull-right text-right\", \"data-toggle\": \"dropdown\", \"aria-haspopup\": \"true\",\n \"aria-expanded\": \"false\" },\n \"Show \",\n _react2.default.createElement(\"span\", { className: \"caret\" })\n ),\n _react2.default.createElement(\n \"ul\",\n { className: \"dropdown-menu\" },\n this.renderTaskFilterListItem(TASK_FILTER.ALL, \"All\"),\n this.renderTaskFilterListItem(TASK_FILTER.PLANNED, \"Planned\"),\n this.renderTaskFilterListItem(TASK_FILTER.RUNNING, \"Running\"),\n this.renderTaskFilterListItem(TASK_FILTER.FINISHED, \"Finished\"),\n this.renderTaskFilterListItem(TASK_FILTER.FAILED, \"Aborted/Canceled/Failed\")\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n \"\\xA0\\xA0\",\n this.renderTaskRefreshButton()\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(TaskList, { key: this.state.query.queryId, tasks: tasks })\n )\n )\n );\n }\n }, {\n key: \"renderStages\",\n value: function renderStages() {\n if (this.state.lastSnapshotStage === null) {\n return;\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-9\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Stages\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-3\" },\n _react2.default.createElement(\n \"table\",\n { className: \"header-inline-links\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n this.renderStageRefreshButton()\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(StageList, { key: this.state.query.queryId, outputStage: this.state.lastSnapshotStage })\n )\n )\n );\n }\n }, {\n key: \"renderPreparedQuery\",\n value: function renderPreparedQuery() {\n var query = this.state.query;\n if (!query.hasOwnProperty('preparedQuery') || query.preparedQuery === null) {\n return;\n }\n\n return _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Prepared Query\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#prepared-query-text\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"pre\",\n { id: \"prepared-query\" },\n _react2.default.createElement(\n \"code\",\n { className: \"lang-sql\", id: \"prepared-query-text\" },\n query.preparedQuery\n )\n )\n );\n }\n }, {\n key: \"renderSessionProperties\",\n value: function renderSessionProperties() {\n var query = this.state.query;\n\n var properties = [];\n for (var property in query.session.systemProperties) {\n if (query.session.systemProperties.hasOwnProperty(property)) {\n properties.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n property + \"=\" + query.session.systemProperties[property],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n\n for (var catalog in query.session.catalogProperties) {\n if (query.session.catalogProperties.hasOwnProperty(catalog)) {\n for (var _property in query.session.catalogProperties[catalog]) {\n if (query.session.catalogProperties[catalog].hasOwnProperty(_property)) {\n properties.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n catalog + \".\" + _property + \"=\" + query.session.catalogProperties[catalog][_property],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n }\n }\n\n return properties;\n }\n }, {\n key: \"renderResourceEstimates\",\n value: function renderResourceEstimates() {\n var query = this.state.query;\n var estimates = query.session.resourceEstimates;\n var renderedEstimates = [];\n\n for (var resource in estimates) {\n if (estimates.hasOwnProperty(resource)) {\n var upperChars = resource.match(/([A-Z])/g) || [];\n var snakeCased = resource;\n for (var i = 0, n = upperChars.length; i < n; i++) {\n snakeCased = snakeCased.replace(new RegExp(upperChars[i]), '_' + upperChars[i].toLowerCase());\n }\n\n renderedEstimates.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n snakeCased + \"=\" + query.session.resourceEstimates[resource],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n\n return renderedEstimates;\n }\n }, {\n key: \"renderWarningInfo\",\n value: function renderWarningInfo() {\n var query = this.state.query;\n if (query.warnings.length > 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Warnings\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\", id: \"warnings-table\" },\n query.warnings.map(function (warning) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n warning.warningCode.name\n ),\n _react2.default.createElement(\n \"td\",\n null,\n warning.message\n )\n );\n })\n )\n )\n );\n } else {\n return null;\n }\n }\n }, {\n key: \"renderFailureInfo\",\n value: function renderFailureInfo() {\n var query = this.state.query;\n if (query.failureInfo) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Error Information\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Error Type\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.errorType\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Error Code\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.errorCode.name + \" (\" + this.state.query.errorCode.code + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Stack Trace\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#stack-trace\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n _react2.default.createElement(\n \"pre\",\n { id: \"stack-trace\" },\n QueryDetail.formatStackTrace(query.failureInfo)\n )\n )\n )\n )\n )\n )\n );\n } else {\n return \"\";\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var query = this.state.query;\n\n if (query === null || this.state.initialized === false) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(_QueryHeader.QueryHeader, { query: query }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Session\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"User\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"query-user\" },\n query.session.user\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#query-user\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Principal\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.session.principal\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Source\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.session.source\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Catalog\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.catalog\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Schema\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.schema\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Time zone\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.timeZone\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Client Address\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.remoteUserAddress\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Client Tags\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.clientTags.join(\", \")\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Session Properties\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n this.renderSessionProperties()\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Resource Estimates\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n this.renderResourceEstimates()\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Execution\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Resource Group\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.resourceGroupId ? query.resourceGroupId.join(\".\") : \"n/a\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Submission Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatShortDateTime)(new Date(query.queryStats.createTime))\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Completion Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.endTime ? (0, _utils.formatShortDateTime)(new Date(query.queryStats.endTime)) : \"\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Elapsed Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.elapsedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Queued Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.queuedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Analysis Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.analysisTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Planning Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.planningTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Execution Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.executionTime\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Resource Utilization Summary\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"CPU Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalCpuTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Scheduled Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalScheduledTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.processedInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.processedInputDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Input Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.physicalInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Input Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.physicalInputDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Input Read Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.physicalInputReadTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Internal Network Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.internalNetworkInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Internal Network Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.internalNetworkInputDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak User Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.peakUserMemoryReservation)\n )\n ),\n (0, _utils.parseDataSize)(query.queryStats.peakRevocableMemoryReservation) > 0 && _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak Revocable Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.peakRevocableMemoryReservation)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak Total Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.peakTotalMemoryReservation)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Memory Pool\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.memoryPool\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Cumulative User Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatDataSizeBytes)(query.queryStats.cumulativeUserMemory / 1000.0) + \" seconds\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Cumulative System Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatDataSizeBytes)(query.queryStats.cumulativeSystemMemory / 1000.0) + \" seconds\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Output Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.outputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Output Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.outputDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Written Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.writtenPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Logical Written Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.logicalWrittenDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Written Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.physicalWrittenDataSize)\n )\n ),\n (0, _utils.parseDataSize)(query.queryStats.spilledDataSize) > 0 && _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Spilled Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.spilledDataSize)\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Timeline\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Parallelism\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"cpu-time-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.cpuTimeRate[this.state.cpuTimeRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Scheduled Time/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"scheduled-time-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.scheduledTimeRate[this.state.scheduledTimeRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Rows/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"row-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.rowInputRate[this.state.rowInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Bytes/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"byte-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.byteInputRate[this.state.byteInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Input Bytes/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"physical-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.physicalInputRate[this.state.physicalInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Memory Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"reserved-memory-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.reservedMemory[this.state.reservedMemory.length - 1])\n )\n )\n )\n )\n )\n )\n )\n ),\n this.renderWarningInfo(),\n this.renderFailureInfo(),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Query\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#query-text\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"pre\",\n { id: \"query\" },\n _react2.default.createElement(\n \"code\",\n { className: \"lang-sql\", id: \"query-text\" },\n query.query\n )\n )\n ),\n this.renderPreparedQuery()\n ),\n this.renderStages(),\n this.renderTasks()\n );\n }\n }], [{\n key: \"formatStackTrace\",\n value: function formatStackTrace(info) {\n return QueryDetail.formatStackTraceHelper(info, [], \"\", \"\");\n }\n }, {\n key: \"formatStackTraceHelper\",\n value: function formatStackTraceHelper(info, parentStack, prefix, linePrefix) {\n var s = linePrefix + prefix + QueryDetail.failureInfoToString(info) + \"\\n\";\n\n if (info.stack) {\n var sharedStackFrames = 0;\n if (parentStack !== null) {\n sharedStackFrames = QueryDetail.countSharedStackFrames(info.stack, parentStack);\n }\n\n for (var i = 0; i < info.stack.length - sharedStackFrames; i++) {\n s += linePrefix + \"\\tat \" + info.stack[i] + \"\\n\";\n }\n if (sharedStackFrames !== 0) {\n s += linePrefix + \"\\t... \" + sharedStackFrames + \" more\" + \"\\n\";\n }\n }\n\n if (info.suppressed) {\n for (var _i3 = 0; _i3 < info.suppressed.length; _i3++) {\n s += QueryDetail.formatStackTraceHelper(info.suppressed[_i3], info.stack, \"Suppressed: \", linePrefix + \"\\t\");\n }\n }\n\n if (info.cause) {\n s += QueryDetail.formatStackTraceHelper(info.cause, info.stack, \"Caused by: \", linePrefix);\n }\n\n return s;\n }\n }, {\n key: \"countSharedStackFrames\",\n value: function countSharedStackFrames(stack, parentStack) {\n var n = 0;\n var minStackLength = Math.min(stack.length, parentStack.length);\n while (n < minStackLength && stack[stack.length - 1 - n] === parentStack[parentStack.length - 1 - n]) {\n n++;\n }\n return n;\n }\n }, {\n key: \"failureInfoToString\",\n value: function failureInfoToString(t) {\n return t.message !== null ? t.type + \": \" + t.message : t.type;\n }\n }]);\n\n return QueryDetail;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/QueryDetail.jsx?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.QueryDetail = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _reactable = __webpack_require__(/*! reactable */ \"./node_modules/reactable/lib/reactable.js\");\n\nvar _reactable2 = _interopRequireDefault(_reactable);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nvar Table = _reactable2.default.Table,\n Thead = _reactable2.default.Thead,\n Th = _reactable2.default.Th,\n Tr = _reactable2.default.Tr,\n Td = _reactable2.default.Td;\n\nvar TaskList = function (_React$Component) {\n _inherits(TaskList, _React$Component);\n\n function TaskList() {\n _classCallCheck(this, TaskList);\n\n return _possibleConstructorReturn(this, (TaskList.__proto__ || Object.getPrototypeOf(TaskList)).apply(this, arguments));\n }\n\n _createClass(TaskList, [{\n key: \"render\",\n value: function render() {\n var tasks = this.props.tasks;\n\n if (tasks === undefined || tasks.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"No threads in the selected group\"\n )\n )\n );\n }\n\n var showPortNumbers = TaskList.showPortNumbers(tasks);\n\n var renderedTasks = tasks.map(function (task) {\n var elapsedTime = (0, _utils.parseDuration)(task.stats.elapsedTime);\n if (elapsedTime === 0) {\n elapsedTime = Date.now() - Date.parse(task.stats.createTime);\n }\n\n return _react2.default.createElement(\n Tr,\n { key: task.taskStatus.taskId },\n _react2.default.createElement(\n Td,\n { column: \"id\", value: task.taskStatus.taskId },\n _react2.default.createElement(\n \"a\",\n { href: \"/ui/api/worker/\" + task.taskStatus.nodeId + \"/task/\" + task.taskStatus.taskId + \"?pretty\" },\n (0, _utils.getTaskIdSuffix)(task.taskStatus.taskId)\n )\n ),\n _react2.default.createElement(\n Td,\n { column: \"host\", value: (0, _utils.getHostname)(task.taskStatus.self) },\n _react2.default.createElement(\n \"a\",\n { href: \"worker.html?\" + task.taskStatus.nodeId, className: \"font-light\", target: \"_blank\" },\n showPortNumbers ? (0, _utils.getHostAndPort)(task.taskStatus.self) : (0, _utils.getHostname)(task.taskStatus.self)\n )\n ),\n _react2.default.createElement(\n Td,\n { column: \"state\", value: TaskList.formatState(task.taskStatus.state, task.stats.fullyBlocked) },\n TaskList.formatState(task.taskStatus.state, task.stats.fullyBlocked)\n ),\n _react2.default.createElement(\n Td,\n { column: \"rows\", value: task.stats.rawInputPositions },\n (0, _utils.formatCount)(task.stats.rawInputPositions)\n ),\n _react2.default.createElement(\n Td,\n { column: \"rowsSec\", value: (0, _utils.computeRate)(task.stats.rawInputPositions, elapsedTime) },\n (0, _utils.formatCount)((0, _utils.computeRate)(task.stats.rawInputPositions, elapsedTime))\n ),\n _react2.default.createElement(\n Td,\n { column: \"bytes\", value: (0, _utils.parseDataSize)(task.stats.rawInputDataSize) },\n (0, _utils.formatDataSizeBytes)((0, _utils.parseDataSize)(task.stats.rawInputDataSize))\n ),\n _react2.default.createElement(\n Td,\n { column: \"bytesSec\", value: (0, _utils.computeRate)((0, _utils.parseDataSize)(task.stats.rawInputDataSize), elapsedTime) },\n (0, _utils.formatDataSizeBytes)((0, _utils.computeRate)((0, _utils.parseDataSize)(task.stats.rawInputDataSize), elapsedTime))\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsPending\", value: task.stats.queuedDrivers },\n task.stats.queuedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsRunning\", value: task.stats.runningDrivers },\n task.stats.runningDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsBlocked\", value: task.stats.blockedDrivers },\n task.stats.blockedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsDone\", value: task.stats.completedDrivers },\n task.stats.completedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"elapsedTime\", value: (0, _utils.parseDuration)(task.stats.elapsedTime) },\n task.stats.elapsedTime\n ),\n _react2.default.createElement(\n Td,\n { column: \"cpuTime\", value: (0, _utils.parseDuration)(task.stats.totalCpuTime) },\n task.stats.totalCpuTime\n ),\n _react2.default.createElement(\n Td,\n { column: \"bufferedBytes\", value: task.outputBuffers.totalBufferedBytes },\n (0, _utils.formatDataSizeBytes)(task.outputBuffers.totalBufferedBytes)\n )\n );\n });\n\n return _react2.default.createElement(\n Table,\n { id: \"tasks\", className: \"table table-striped sortable\", sortable: [{\n column: 'id',\n sortFunction: TaskList.compareTaskId\n }, 'host', 'state', 'splitsPending', 'splitsRunning', 'splitsBlocked', 'splitsDone', 'rows', 'rowsSec', 'bytes', 'bytesSec', 'elapsedTime', 'cpuTime', 'bufferedBytes'],\n defaultSort: { column: 'id', direction: 'asc' } },\n _react2.default.createElement(\n Thead,\n null,\n _react2.default.createElement(\n Th,\n { column: \"id\" },\n \"ID\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"host\" },\n \"Host\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"state\" },\n \"State\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsPending\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-pause\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Pending splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsRunning\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-play\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Running splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsBlocked\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-bookmark\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Blocked splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsDone\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-ok\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Completed splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"rows\" },\n \"Rows\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"rowsSec\" },\n \"Rows/s\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bytes\" },\n \"Bytes\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bytesSec\" },\n \"Bytes/s\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"elapsedTime\" },\n \"Elapsed\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"cpuTime\" },\n \"CPU Time\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bufferedBytes\" },\n \"Buffered\"\n )\n ),\n renderedTasks\n );\n }\n }], [{\n key: \"removeQueryId\",\n value: function removeQueryId(id) {\n var pos = id.indexOf('.');\n if (pos !== -1) {\n return id.substring(pos + 1);\n }\n return id;\n }\n }, {\n key: \"compareTaskId\",\n value: function compareTaskId(taskA, taskB) {\n var taskIdArrA = TaskList.removeQueryId(taskA).split(\".\");\n var taskIdArrB = TaskList.removeQueryId(taskB).split(\".\");\n\n if (taskIdArrA.length > taskIdArrB.length) {\n return 1;\n }\n for (var i = 0; i < taskIdArrA.length; i++) {\n var anum = Number.parseInt(taskIdArrA[i]);\n var bnum = Number.parseInt(taskIdArrB[i]);\n if (anum !== bnum) {\n return anum > bnum ? 1 : -1;\n }\n }\n\n return 0;\n }\n }, {\n key: \"showPortNumbers\",\n value: function showPortNumbers(tasks) {\n // check if any host has multiple port numbers\n var hostToPortNumber = {};\n for (var i = 0; i < tasks.length; i++) {\n var taskUri = tasks[i].taskStatus.self;\n var hostname = (0, _utils.getHostname)(taskUri);\n var port = (0, _utils.getPort)(taskUri);\n if (hostname in hostToPortNumber && hostToPortNumber[hostname] !== port) {\n return true;\n }\n hostToPortNumber[hostname] = port;\n }\n\n return false;\n }\n }, {\n key: \"formatState\",\n value: function formatState(state, fullyBlocked) {\n if (fullyBlocked && state === \"RUNNING\") {\n return \"BLOCKED\";\n } else {\n return state;\n }\n }\n }]);\n\n return TaskList;\n}(_react2.default.Component);\n\nvar BAR_CHART_WIDTH = 800;\n\nvar BAR_CHART_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#8997B3',\n chartRangeMin: 0,\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: 'Task {{offset:offset}} - {{value}}',\n disableHiddenCheck: true\n};\n\nvar HISTOGRAM_WIDTH = 175;\n\nvar HISTOGRAM_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#747F96',\n zeroAxis: true,\n chartRangeMin: 0,\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: '{{offset:offset}} -- {{value}} tasks',\n disableHiddenCheck: true\n};\n\nvar StageSummary = function (_React$Component2) {\n _inherits(StageSummary, _React$Component2);\n\n function StageSummary(props) {\n _classCallCheck(this, StageSummary);\n\n var _this2 = _possibleConstructorReturn(this, (StageSummary.__proto__ || Object.getPrototypeOf(StageSummary)).call(this, props));\n\n _this2.state = {\n expanded: false,\n lastRender: null\n };\n return _this2;\n }\n\n _createClass(StageSummary, [{\n key: \"getExpandedIcon\",\n value: function getExpandedIcon() {\n return this.state.expanded ? \"glyphicon-chevron-up\" : \"glyphicon-chevron-down\";\n }\n }, {\n key: \"getExpandedStyle\",\n value: function getExpandedStyle() {\n return this.state.expanded ? {} : { display: \"none\" };\n }\n }, {\n key: \"toggleExpanded\",\n value: function toggleExpanded() {\n this.setState({\n expanded: !this.state.expanded\n });\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n var stage = this.props.stage;\n var numTasks = stage.tasks.length;\n\n // sort the x-axis\n stage.tasks.sort(function (taskA, taskB) {\n return (0, _utils.getTaskNumber)(taskA.taskStatus.taskId) - (0, _utils.getTaskNumber)(taskB.taskStatus.taskId);\n });\n\n var scheduledTimes = stage.tasks.map(function (task) {\n return (0, _utils.parseDuration)(task.stats.totalScheduledTime);\n });\n var cpuTimes = stage.tasks.map(function (task) {\n return (0, _utils.parseDuration)(task.stats.totalCpuTime);\n });\n\n // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts\n if (this.state.lastRender === null || Date.now() - this.state.lastRender >= 1000) {\n var renderTimestamp = Date.now();\n var stageId = (0, _utils.getStageNumber)(stage.stageId);\n\n StageSummary.renderHistogram('#scheduled-time-histogram-' + stageId, scheduledTimes, _utils.formatDuration);\n StageSummary.renderHistogram('#cpu-time-histogram-' + stageId, cpuTimes, _utils.formatDuration);\n\n if (this.state.expanded) {\n // this needs to be a string otherwise it will also be passed to numberFormatter\n var tooltipValueLookups = { 'offset': {} };\n for (var i = 0; i < numTasks; i++) {\n tooltipValueLookups['offset'][i] = (0, _utils.getStageNumber)(stage.stageId) + \".\" + i;\n }\n\n var stageBarChartProperties = $.extend({}, BAR_CHART_PROPERTIES, { barWidth: BAR_CHART_WIDTH / numTasks, tooltipValueLookups: tooltipValueLookups });\n\n $('#scheduled-time-bar-chart-' + stageId).sparkline(scheduledTimes, $.extend({}, stageBarChartProperties, { numberFormatter: _utils.formatDuration }));\n $('#cpu-time-bar-chart-' + stageId).sparkline(cpuTimes, $.extend({}, stageBarChartProperties, { numberFormatter: _utils.formatDuration }));\n }\n\n this.setState({\n lastRender: renderTimestamp\n });\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n if (stage === undefined || !stage.hasOwnProperty('plan')) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Information about this stage is unavailable.\"\n )\n );\n }\n\n var totalBufferedBytes = stage.tasks.map(function (task) {\n return task.outputBuffers.totalBufferedBytes;\n }).reduce(function (a, b) {\n return a + b;\n }, 0);\n\n var stageId = (0, _utils.getStageNumber)(stage.stageId);\n\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-id\" },\n _react2.default.createElement(\n \"div\",\n { className: \"stage-state-color\", style: { borderLeftColor: (0, _utils.getStageStateColor)(stage) } },\n stageId\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"table single-stage-table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-time\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Time\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Scheduled\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalScheduledTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalBlockedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"CPU\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalCpuTime\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-memory\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Memory\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Cumulative\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.formatDataSizeBytes)(stage.stageStats.cumulativeUserMemory / 1000)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Current\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.parseAndFormatDataSize)(stage.stageStats.userMemoryReservation)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Buffers\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.formatDataSize)(totalBufferedBytes)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Peak\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.parseAndFormatDataSize)(stage.stageStats.peakUserMemoryReservation)\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-tasks\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Tasks\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Pending\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.taskStatus.state === \"PLANNED\";\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Running\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.taskStatus.state === \"RUNNING\";\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.stats.fullyBlocked;\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Total\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.length\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table histogram-table\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-chart-header\" },\n \"Scheduled Time Skew\"\n )\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"histogram-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"histogram\", id: \"scheduled-time-histogram-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table histogram-table\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-chart-header\" },\n \"CPU Time Skew\"\n )\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"histogram-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"histogram\", id: \"cpu-time-histogram-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"expand-charts-container\" },\n _react2.default.createElement(\n \"a\",\n { onClick: this.toggleExpanded.bind(this), className: \"expand-charts-button\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon \" + this.getExpandedIcon(), style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\", title: \"More\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"expanded-chart\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title expanded-chart-title\" },\n \"Task Scheduled Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"bar-chart-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"bar-chart\", id: \"scheduled-time-bar-chart-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"expanded-chart\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title expanded-chart-title\" },\n \"Task CPU Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"bar-chart-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"bar-chart\", id: \"cpu-time-bar-chart-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n )\n )\n )\n )\n )\n );\n }\n }], [{\n key: \"renderHistogram\",\n value: function renderHistogram(histogramId, inputData, numberFormatter) {\n var numBuckets = Math.min(HISTOGRAM_WIDTH, Math.sqrt(inputData.length));\n var dataMin = Math.min.apply(null, inputData);\n var dataMax = Math.max.apply(null, inputData);\n var bucketSize = (dataMax - dataMin) / numBuckets;\n\n var histogramData = [];\n if (bucketSize === 0) {\n histogramData = [inputData.length];\n } else {\n for (var i = 0; i < numBuckets + 1; i++) {\n histogramData.push(0);\n }\n\n for (var _i in inputData) {\n var dataPoint = inputData[_i];\n var bucket = Math.floor((dataPoint - dataMin) / bucketSize);\n histogramData[bucket] = histogramData[bucket] + 1;\n }\n }\n\n var tooltipValueLookups = { 'offset': {} };\n for (var _i2 = 0; _i2 < histogramData.length; _i2++) {\n tooltipValueLookups['offset'][_i2] = numberFormatter(dataMin + _i2 * bucketSize) + \"-\" + numberFormatter(dataMin + (_i2 + 1) * bucketSize);\n }\n\n var stageHistogramProperties = $.extend({}, HISTOGRAM_PROPERTIES, { barWidth: HISTOGRAM_WIDTH / histogramData.length, tooltipValueLookups: tooltipValueLookups });\n $(histogramId).sparkline(histogramData, stageHistogramProperties);\n }\n }]);\n\n return StageSummary;\n}(_react2.default.Component);\n\nvar StageList = function (_React$Component3) {\n _inherits(StageList, _React$Component3);\n\n function StageList() {\n _classCallCheck(this, StageList);\n\n return _possibleConstructorReturn(this, (StageList.__proto__ || Object.getPrototypeOf(StageList)).apply(this, arguments));\n }\n\n _createClass(StageList, [{\n key: \"getStages\",\n value: function getStages(stage) {\n if (stage === undefined || !stage.hasOwnProperty('subStages')) {\n return [];\n }\n\n return [].concat.apply(stage, stage.subStages.map(this.getStages, this));\n }\n }, {\n key: \"render\",\n value: function render() {\n var stages = this.getStages(this.props.outputStage);\n\n if (stages === undefined || stages.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n \"No stage information available.\"\n )\n );\n }\n\n var renderedStages = stages.map(function (stage) {\n return _react2.default.createElement(StageSummary, { key: stage.stageId, stage: stage });\n });\n\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\", id: \"stage-list\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n renderedStages\n )\n )\n )\n );\n }\n }]);\n\n return StageList;\n}(_react2.default.Component);\n\nvar SMALL_SPARKLINE_PROPERTIES = {\n width: '100%',\n height: '57px',\n fillColor: '#3F4552',\n lineColor: '#747F96',\n spotColor: '#1EDCFF',\n tooltipClassname: 'sparkline-tooltip',\n disableHiddenCheck: true\n};\n\nvar TASK_FILTER = {\n ALL: function ALL() {\n return true;\n },\n PLANNED: function PLANNED(state) {\n return state === 'PLANNED';\n },\n RUNNING: function RUNNING(state) {\n return state === 'RUNNING';\n },\n FINISHED: function FINISHED(state) {\n return state === 'FINISHED';\n },\n FAILED: function FAILED(state) {\n return state === 'FAILED' || state === 'ABORTED' || state === 'CANCELED';\n }\n};\n\nvar QueryDetail = exports.QueryDetail = function (_React$Component4) {\n _inherits(QueryDetail, _React$Component4);\n\n function QueryDetail(props) {\n _classCallCheck(this, QueryDetail);\n\n var _this4 = _possibleConstructorReturn(this, (QueryDetail.__proto__ || Object.getPrototypeOf(QueryDetail)).call(this, props));\n\n _this4.state = {\n query: null,\n lastSnapshotStages: null,\n lastSnapshotTasks: null,\n\n lastScheduledTime: 0,\n lastCpuTime: 0,\n lastRowInput: 0,\n lastByteInput: 0,\n lastPhysicalInput: 0,\n lastPhysicalTime: 0,\n\n scheduledTimeRate: [],\n cpuTimeRate: [],\n rowInputRate: [],\n byteInputRate: [],\n physicalInputRate: [],\n\n reservedMemory: [],\n\n initialized: false,\n queryEnded: false,\n renderingEnded: false,\n\n lastRefresh: null,\n lastRender: null,\n\n stageRefresh: true,\n taskRefresh: true,\n\n taskFilter: TASK_FILTER.ALL\n };\n\n _this4.refreshLoop = _this4.refreshLoop.bind(_this4);\n return _this4;\n }\n\n _createClass(QueryDetail, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.queryEnded) {\n // task.info-update-interval is set to 3 seconds by default\n this.timeoutId = setTimeout(this.refreshLoop, 3000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this5 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n var queryId = (0, _utils.getFirstParameter)(window.location.search);\n $.get('/ui/api/query/' + queryId, function (query) {\n var lastSnapshotStages = this.state.lastSnapshotStage;\n if (this.state.stageRefresh) {\n lastSnapshotStages = query.outputStage;\n }\n var lastSnapshotTasks = this.state.lastSnapshotTasks;\n if (this.state.taskRefresh) {\n lastSnapshotTasks = query.outputStage;\n }\n\n var lastRefresh = this.state.lastRefresh;\n var lastScheduledTime = this.state.lastScheduledTime;\n var lastCpuTime = this.state.lastCpuTime;\n var lastRowInput = this.state.lastRowInput;\n var lastByteInput = this.state.lastByteInput;\n var lastPhysicalInput = this.state.lastPhysicalInput;\n var lastPhysicalTime = this.state.lastPhysicalTime;\n var alreadyEnded = this.state.queryEnded;\n var nowMillis = Date.now();\n\n this.setState({\n query: query,\n lastSnapshotStage: lastSnapshotStages,\n lastSnapshotTasks: lastSnapshotTasks,\n\n lastPhysicalTime: (0, _utils.parseDuration)(query.queryStats.physicalInputReadTime),\n lastScheduledTime: (0, _utils.parseDuration)(query.queryStats.totalScheduledTime),\n lastCpuTime: (0, _utils.parseDuration)(query.queryStats.totalCpuTime),\n lastRowInput: query.queryStats.processedInputPositions,\n lastByteInput: (0, _utils.parseDataSize)(query.queryStats.processedInputDataSize),\n lastPhysicalInput: (0, _utils.parseDataSize)(query.queryStats.physicalInputDataSize),\n\n initialized: true,\n queryEnded: !!query.finalQueryInfo,\n\n lastRefresh: nowMillis\n });\n\n // i.e. don't show sparklines if we've already decided not to update or if we don't have one previous measurement\n if (alreadyEnded || lastRefresh === null && query.state === \"RUNNING\") {\n this.resetTimer();\n return;\n }\n\n if (lastRefresh === null) {\n lastRefresh = nowMillis - (0, _utils.parseDuration)(query.queryStats.elapsedTime);\n }\n\n var elapsedSecsSinceLastRefresh = (nowMillis - lastRefresh) / 1000.0;\n if (elapsedSecsSinceLastRefresh >= 0) {\n var currentScheduledTimeRate = ((0, _utils.parseDuration)(query.queryStats.totalScheduledTime) - lastScheduledTime) / (elapsedSecsSinceLastRefresh * 1000);\n var currentCpuTimeRate = ((0, _utils.parseDuration)(query.queryStats.totalCpuTime) - lastCpuTime) / (elapsedSecsSinceLastRefresh * 1000);\n var currentPhysicalReadTime = ((0, _utils.parseDuration)(query.queryStats.physicalInputReadTime) - lastPhysicalTime) / 1000;\n var currentRowInputRate = (query.queryStats.processedInputPositions - lastRowInput) / elapsedSecsSinceLastRefresh;\n var currentByteInputRate = ((0, _utils.parseDataSize)(query.queryStats.processedInputDataSize) - lastByteInput) / elapsedSecsSinceLastRefresh;\n var currentPhysicalInputRate = currentPhysicalReadTime > 0 ? ((0, _utils.parseDataSize)(query.queryStats.physicalInputDataSize) - lastPhysicalInput) / currentPhysicalReadTime : 0;\n\n this.setState({\n scheduledTimeRate: (0, _utils.addToHistory)(currentScheduledTimeRate, this.state.scheduledTimeRate),\n cpuTimeRate: (0, _utils.addToHistory)(currentCpuTimeRate, this.state.cpuTimeRate),\n rowInputRate: (0, _utils.addToHistory)(currentRowInputRate, this.state.rowInputRate),\n byteInputRate: (0, _utils.addToHistory)(currentByteInputRate, this.state.byteInputRate),\n reservedMemory: (0, _utils.addToHistory)((0, _utils.parseDataSize)(query.queryStats.totalMemoryReservation), this.state.reservedMemory),\n physicalInputRate: (0, _utils.addToHistory)(currentPhysicalInputRate, this.state.physicalInputRate)\n });\n }\n this.resetTimer();\n }.bind(this)).fail(function () {\n _this5.setState({\n initialized: true\n });\n _this5.resetTimer();\n });\n }\n }, {\n key: \"handleTaskRefreshClick\",\n value: function handleTaskRefreshClick() {\n if (this.state.taskRefresh) {\n this.setState({\n taskRefresh: false,\n lastSnapshotTasks: this.state.query.outputStage\n });\n } else {\n this.setState({\n taskRefresh: true\n });\n }\n }\n }, {\n key: \"renderTaskRefreshButton\",\n value: function renderTaskRefreshButton() {\n if (this.state.taskRefresh) {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleTaskRefreshClick.bind(this) },\n \"Auto-Refresh: On\"\n );\n } else {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleTaskRefreshClick.bind(this) },\n \"Auto-Refresh: Off\"\n );\n }\n }\n }, {\n key: \"handleStageRefreshClick\",\n value: function handleStageRefreshClick() {\n if (this.state.stageRefresh) {\n this.setState({\n stageRefresh: false,\n lastSnapshotStages: this.state.query.outputStage\n });\n } else {\n this.setState({\n stageRefresh: true\n });\n }\n }\n }, {\n key: \"renderStageRefreshButton\",\n value: function renderStageRefreshButton() {\n if (this.state.stageRefresh) {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleStageRefreshClick.bind(this) },\n \"Auto-Refresh: On\"\n );\n } else {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleStageRefreshClick.bind(this) },\n \"Auto-Refresh: Off\"\n );\n }\n }\n }, {\n key: \"renderTaskFilterListItem\",\n value: function renderTaskFilterListItem(taskFilter, taskFilterText) {\n return _react2.default.createElement(\n \"li\",\n null,\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: this.state.taskFilter === taskFilter ? \"selected\" : \"\", onClick: this.handleTaskFilterClick.bind(this, taskFilter) },\n taskFilterText\n )\n );\n }\n }, {\n key: \"handleTaskFilterClick\",\n value: function handleTaskFilterClick(filter, event) {\n this.setState({\n taskFilter: filter\n });\n event.preventDefault();\n }\n }, {\n key: \"getTasksFromStage\",\n value: function getTasksFromStage(stage) {\n if (stage === undefined || !stage.hasOwnProperty('subStages') || !stage.hasOwnProperty('tasks')) {\n return [];\n }\n\n return [].concat.apply(stage.tasks, stage.subStages.map(this.getTasksFromStage, this));\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop();\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts\n if (this.state.lastRender === null || Date.now() - this.state.lastRender >= 1000 || this.state.ended && !this.state.renderingEnded) {\n var renderTimestamp = Date.now();\n $('#scheduled-time-rate-sparkline').sparkline(this.state.scheduledTimeRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, {\n chartRangeMin: 0,\n numberFormatter: _utils.precisionRound\n }));\n $('#cpu-time-rate-sparkline').sparkline(this.state.cpuTimeRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#row-input-rate-sparkline').sparkline(this.state.rowInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatCount }));\n $('#byte-input-rate-sparkline').sparkline(this.state.byteInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n $('#reserved-memory-sparkline').sparkline(this.state.reservedMemory, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n $('#physical-input-rate-sparkline').sparkline(this.state.physicalInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n\n if (this.state.lastRender === null) {\n $('#query').each(function (i, block) {\n hljs.highlightBlock(block);\n });\n\n $('#prepared-query').each(function (i, block) {\n hljs.highlightBlock(block);\n });\n }\n\n this.setState({\n renderingEnded: this.state.ended,\n lastRender: renderTimestamp\n });\n }\n\n $('[data-toggle=\"tooltip\"]').tooltip();\n new window.ClipboardJS('.copy-button');\n }\n }, {\n key: \"renderTasks\",\n value: function renderTasks() {\n var _this6 = this;\n\n if (this.state.lastSnapshotTasks === null) {\n return;\n }\n\n var tasks = this.getTasksFromStage(this.state.lastSnapshotTasks).filter(function (task) {\n return _this6.state.taskFilter(task.taskStatus.state);\n }, this);\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-9\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Tasks\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-3\" },\n _react2.default.createElement(\n \"table\",\n { className: \"header-inline-links\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"input-group-btn text-right\" },\n _react2.default.createElement(\n \"button\",\n { type: \"button\", className: \"btn btn-default dropdown-toggle pull-right text-right\", \"data-toggle\": \"dropdown\", \"aria-haspopup\": \"true\",\n \"aria-expanded\": \"false\" },\n \"Show \",\n _react2.default.createElement(\"span\", { className: \"caret\" })\n ),\n _react2.default.createElement(\n \"ul\",\n { className: \"dropdown-menu\" },\n this.renderTaskFilterListItem(TASK_FILTER.ALL, \"All\"),\n this.renderTaskFilterListItem(TASK_FILTER.PLANNED, \"Planned\"),\n this.renderTaskFilterListItem(TASK_FILTER.RUNNING, \"Running\"),\n this.renderTaskFilterListItem(TASK_FILTER.FINISHED, \"Finished\"),\n this.renderTaskFilterListItem(TASK_FILTER.FAILED, \"Aborted/Canceled/Failed\")\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n \"\\xA0\\xA0\",\n this.renderTaskRefreshButton()\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(TaskList, { key: this.state.query.queryId, tasks: tasks })\n )\n )\n );\n }\n }, {\n key: \"renderStages\",\n value: function renderStages() {\n if (this.state.lastSnapshotStage === null) {\n return;\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-9\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Stages\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-3\" },\n _react2.default.createElement(\n \"table\",\n { className: \"header-inline-links\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n this.renderStageRefreshButton()\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(StageList, { key: this.state.query.queryId, outputStage: this.state.lastSnapshotStage })\n )\n )\n );\n }\n }, {\n key: \"renderPreparedQuery\",\n value: function renderPreparedQuery() {\n var query = this.state.query;\n if (!query.hasOwnProperty('preparedQuery') || query.preparedQuery === null) {\n return;\n }\n\n return _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Prepared Query\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#prepared-query-text\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"pre\",\n { id: \"prepared-query\" },\n _react2.default.createElement(\n \"code\",\n { className: \"lang-sql\", id: \"prepared-query-text\" },\n query.preparedQuery\n )\n )\n );\n }\n }, {\n key: \"renderSessionProperties\",\n value: function renderSessionProperties() {\n var query = this.state.query;\n\n var properties = [];\n for (var property in query.session.systemProperties) {\n if (query.session.systemProperties.hasOwnProperty(property)) {\n properties.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n property + \"=\" + query.session.systemProperties[property],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n\n for (var catalog in query.session.catalogProperties) {\n if (query.session.catalogProperties.hasOwnProperty(catalog)) {\n for (var _property in query.session.catalogProperties[catalog]) {\n if (query.session.catalogProperties[catalog].hasOwnProperty(_property)) {\n properties.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n catalog + \".\" + _property + \"=\" + query.session.catalogProperties[catalog][_property],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n }\n }\n\n return properties;\n }\n }, {\n key: \"renderResourceEstimates\",\n value: function renderResourceEstimates() {\n var query = this.state.query;\n var estimates = query.session.resourceEstimates;\n var renderedEstimates = [];\n\n for (var resource in estimates) {\n if (estimates.hasOwnProperty(resource)) {\n var upperChars = resource.match(/([A-Z])/g) || [];\n var snakeCased = resource;\n for (var i = 0, n = upperChars.length; i < n; i++) {\n snakeCased = snakeCased.replace(new RegExp(upperChars[i]), '_' + upperChars[i].toLowerCase());\n }\n\n renderedEstimates.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n snakeCased + \"=\" + query.session.resourceEstimates[resource],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n\n return renderedEstimates;\n }\n }, {\n key: \"renderWarningInfo\",\n value: function renderWarningInfo() {\n var query = this.state.query;\n if (query.warnings.length > 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Warnings\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\", id: \"warnings-table\" },\n query.warnings.map(function (warning) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n warning.warningCode.name\n ),\n _react2.default.createElement(\n \"td\",\n null,\n warning.message\n )\n );\n })\n )\n )\n );\n } else {\n return null;\n }\n }\n }, {\n key: \"renderFailureInfo\",\n value: function renderFailureInfo() {\n var query = this.state.query;\n if (query.failureInfo) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Error Information\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Error Type\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.errorType\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Error Code\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.errorCode.name + \" (\" + this.state.query.errorCode.code + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Stack Trace\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#stack-trace\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n _react2.default.createElement(\n \"pre\",\n { id: \"stack-trace\" },\n QueryDetail.formatStackTrace(query.failureInfo)\n )\n )\n )\n )\n )\n )\n );\n } else {\n return \"\";\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var query = this.state.query;\n\n if (query === null || this.state.initialized === false) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(_QueryHeader.QueryHeader, { query: query }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Session\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"User\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"query-user\" },\n query.session.user\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#query-user\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Principal\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.session.principal\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Source\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.session.source\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Catalog\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.catalog\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Schema\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.schema\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Time zone\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.timeZone\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Client Address\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.remoteUserAddress\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Client Tags\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.clientTags.join(\", \")\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Session Properties\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n this.renderSessionProperties()\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Resource Estimates\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n this.renderResourceEstimates()\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Execution\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Resource Group\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.resourceGroupId ? query.resourceGroupId.join(\".\") : \"n/a\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Submission Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatShortDateTime)(new Date(query.queryStats.createTime))\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Completion Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.endTime ? (0, _utils.formatShortDateTime)(new Date(query.queryStats.endTime)) : \"\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Elapsed Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.elapsedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Queued Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.queuedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Analysis Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.analysisTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Planning Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.planningTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Execution Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.executionTime\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Resource Utilization Summary\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"CPU Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalCpuTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Scheduled Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalScheduledTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.processedInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.processedInputDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Input Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.physicalInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Input Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.physicalInputDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Input Read Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.physicalInputReadTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Internal Network Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.internalNetworkInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Internal Network Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.internalNetworkInputDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak User Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.peakUserMemoryReservation)\n )\n ),\n (0, _utils.parseDataSize)(query.queryStats.peakRevocableMemoryReservation) > 0 && _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak Revocable Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.peakRevocableMemoryReservation)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak Total Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.peakTotalMemoryReservation)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Memory Pool\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.memoryPool\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Cumulative User Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatDataSizeBytes)(query.queryStats.cumulativeUserMemory / 1000.0) + \" seconds\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Output Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.outputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Output Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.outputDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Written Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.writtenPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Logical Written Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.logicalWrittenDataSize)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Written Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.physicalWrittenDataSize)\n )\n ),\n (0, _utils.parseDataSize)(query.queryStats.spilledDataSize) > 0 && _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Spilled Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.parseAndFormatDataSize)(query.queryStats.spilledDataSize)\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Timeline\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Parallelism\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"cpu-time-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.cpuTimeRate[this.state.cpuTimeRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Scheduled Time/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"scheduled-time-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.scheduledTimeRate[this.state.scheduledTimeRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Rows/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"row-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.rowInputRate[this.state.rowInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Bytes/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"byte-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.byteInputRate[this.state.byteInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Physical Input Bytes/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"physical-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.physicalInputRate[this.state.physicalInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Memory Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"reserved-memory-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.reservedMemory[this.state.reservedMemory.length - 1])\n )\n )\n )\n )\n )\n )\n )\n ),\n this.renderWarningInfo(),\n this.renderFailureInfo(),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Query\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#query-text\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"pre\",\n { id: \"query\" },\n _react2.default.createElement(\n \"code\",\n { className: \"lang-sql\", id: \"query-text\" },\n query.query\n )\n )\n ),\n this.renderPreparedQuery()\n ),\n this.renderStages(),\n this.renderTasks()\n );\n }\n }], [{\n key: \"formatStackTrace\",\n value: function formatStackTrace(info) {\n return QueryDetail.formatStackTraceHelper(info, [], \"\", \"\");\n }\n }, {\n key: \"formatStackTraceHelper\",\n value: function formatStackTraceHelper(info, parentStack, prefix, linePrefix) {\n var s = linePrefix + prefix + QueryDetail.failureInfoToString(info) + \"\\n\";\n\n if (info.stack) {\n var sharedStackFrames = 0;\n if (parentStack !== null) {\n sharedStackFrames = QueryDetail.countSharedStackFrames(info.stack, parentStack);\n }\n\n for (var i = 0; i < info.stack.length - sharedStackFrames; i++) {\n s += linePrefix + \"\\tat \" + info.stack[i] + \"\\n\";\n }\n if (sharedStackFrames !== 0) {\n s += linePrefix + \"\\t... \" + sharedStackFrames + \" more\" + \"\\n\";\n }\n }\n\n if (info.suppressed) {\n for (var _i3 = 0; _i3 < info.suppressed.length; _i3++) {\n s += QueryDetail.formatStackTraceHelper(info.suppressed[_i3], info.stack, \"Suppressed: \", linePrefix + \"\\t\");\n }\n }\n\n if (info.cause) {\n s += QueryDetail.formatStackTraceHelper(info.cause, info.stack, \"Caused by: \", linePrefix);\n }\n\n return s;\n }\n }, {\n key: \"countSharedStackFrames\",\n value: function countSharedStackFrames(stack, parentStack) {\n var n = 0;\n var minStackLength = Math.min(stack.length, parentStack.length);\n while (n < minStackLength && stack[stack.length - 1 - n] === parentStack[parentStack.length - 1 - n]) {\n n++;\n }\n return n;\n }\n }, {\n key: \"failureInfoToString\",\n value: function failureInfoToString(t) {\n return t.message !== null ? t.type + \": \" + t.message : t.type;\n }\n }]);\n\n return QueryDetail;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/QueryDetail.jsx?"); /***/ }), diff --git a/core/trino-main/src/main/resources/webapp/src/components/QueryDetail.jsx b/core/trino-main/src/main/resources/webapp/src/components/QueryDetail.jsx index 71f366aa5ce8..e0e479e57c9b 100644 --- a/core/trino-main/src/main/resources/webapp/src/components/QueryDetail.jsx +++ b/core/trino-main/src/main/resources/webapp/src/components/QueryDetail.jsx @@ -1442,14 +1442,6 @@ export class QueryDetail extends React.Component { {formatDataSizeBytes(query.queryStats.cumulativeUserMemory / 1000.0) + " seconds"} - - - Cumulative System Memory - - - {formatDataSizeBytes(query.queryStats.cumulativeSystemMemory / 1000.0) + " seconds"} - - Output Rows diff --git a/core/trino-main/src/test/java/io/trino/connector/MockConnectorPageSource.java b/core/trino-main/src/test/java/io/trino/connector/MockConnectorPageSource.java index e68526248193..e6330a8079c4 100644 --- a/core/trino-main/src/test/java/io/trino/connector/MockConnectorPageSource.java +++ b/core/trino-main/src/test/java/io/trino/connector/MockConnectorPageSource.java @@ -71,9 +71,9 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return delegate.getSystemMemoryUsage(); + return delegate.getMemoryUsage(); } @Override diff --git a/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java b/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java index c59d53650798..9d76bec04705 100644 --- a/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java +++ b/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java @@ -133,7 +133,6 @@ public BasicQueryInfo getBasicQueryInfo() 15, DataSize.ofBytes(13), 16.0, - 11.0, memoryUsage, memoryUsage, DataSize.ofBytes(19), @@ -187,13 +186,11 @@ public QueryInfo getFullQueryInfo() 16, 17.0, - 11.0, DataSize.ofBytes(18), DataSize.ofBytes(19), DataSize.ofBytes(20), DataSize.ofBytes(21), DataSize.ofBytes(22), - DataSize.ofBytes(30), DataSize.ofBytes(23), DataSize.ofBytes(24), DataSize.ofBytes(25), diff --git a/core/trino-main/src/test/java/io/trino/execution/MockRemoteTaskFactory.java b/core/trino-main/src/test/java/io/trino/execution/MockRemoteTaskFactory.java index 66b1bc02b486..b5b2f4ef3859 100644 --- a/core/trino-main/src/test/java/io/trino/execution/MockRemoteTaskFactory.java +++ b/core/trino-main/src/test/java/io/trino/execution/MockRemoteTaskFactory.java @@ -197,7 +197,6 @@ public MockRemoteTask( SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(DataSize.of(1, GIGABYTE)); QueryContext queryContext = new QueryContext(taskId.getQueryId(), DataSize.of(1, MEGABYTE), - DataSize.of(2, MEGABYTE), Optional.empty(), memoryPool, new TestingGcMonitor(), @@ -264,7 +263,6 @@ public TaskInfo getTaskInfo() DataSize.ofBytes(0), DataSize.ofBytes(0), DataSize.ofBytes(0), - DataSize.ofBytes(0), 0, new Duration(0, MILLISECONDS), INITIAL_DYNAMIC_FILTERS_VERSION, @@ -296,7 +294,6 @@ public TaskStatus getTaskStatus() isOutputBufferOverUtilized, stats.getPhysicalWrittenDataSize(), stats.getUserMemoryReservation(), - stats.getSystemMemoryReservation(), stats.getRevocableMemoryReservation(), 0, new Duration(0, MILLISECONDS), diff --git a/core/trino-main/src/test/java/io/trino/execution/TestMemoryRevokingScheduler.java b/core/trino-main/src/test/java/io/trino/execution/TestMemoryRevokingScheduler.java index 4025859efaa6..285051e6a323 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestMemoryRevokingScheduler.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestMemoryRevokingScheduler.java @@ -317,7 +317,6 @@ private QueryContext getOrCreateQueryContext(QueryId queryId) { return queryContexts.computeIfAbsent(queryId, id -> new QueryContext(id, DataSize.of(1, MEGABYTE), - DataSize.of(2, MEGABYTE), Optional.empty(), memoryPool, new TestingGcMonitor(), diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java index 688f4524a1f7..12f0d1e98fe5 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java @@ -72,10 +72,8 @@ public class TestQueryStats new Duration(122, NANOSECONDS), succinctBytes(124L), succinctBytes(125L), - succinctBytes(126L), succinctBytes(127L), succinctBytes(128L), - succinctBytes(129L), succinctBytes(130L), succinctBytes(131L), Optional.empty(), @@ -113,10 +111,8 @@ public class TestQueryStats new Duration(222, NANOSECONDS), succinctBytes(224L), succinctBytes(225L), - succinctBytes(226L), succinctBytes(227L), succinctBytes(228L), - succinctBytes(229L), succinctBytes(230L), succinctBytes(231L), Optional.empty(), @@ -154,11 +150,9 @@ public class TestQueryStats new Duration(322, NANOSECONDS), succinctBytes(324L), succinctBytes(325L), - succinctBytes(326L), succinctBytes(327L), succinctBytes(328L), succinctBytes(329L), - succinctBytes(330L), succinctBytes(331L), Optional.empty(), null)); @@ -189,13 +183,11 @@ public class TestQueryStats 16, 17.0, - 11.0, DataSize.ofBytes(18), DataSize.ofBytes(19), DataSize.ofBytes(20), DataSize.ofBytes(21), DataSize.ofBytes(22), - DataSize.ofBytes(30), DataSize.ofBytes(23), DataSize.ofBytes(24), DataSize.ofBytes(25), @@ -278,7 +270,6 @@ public static void assertExpectedQueryStats(QueryStats actual) assertEquals(actual.getCompletedDrivers(), 16); assertEquals(actual.getCumulativeUserMemory(), 17.0); - assertEquals(actual.getCumulativeSystemMemory(), 11.0); assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(18)); assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(19)); assertEquals(actual.getTotalMemoryReservation(), DataSize.ofBytes(20)); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestSqlTask.java b/core/trino-main/src/test/java/io/trino/execution/TestSqlTask.java index c15d8f5560c3..41979d19ea79 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestSqlTask.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestSqlTask.java @@ -347,7 +347,6 @@ private SqlTask createInitialTask() QueryContext queryContext = new QueryContext(new QueryId("query"), DataSize.of(1, MEGABYTE), - DataSize.of(2, MEGABYTE), Optional.empty(), new MemoryPool(new MemoryPoolId("test"), DataSize.of(1, GIGABYTE)), new TestingGcMonitor(), diff --git a/core/trino-main/src/test/java/io/trino/execution/TestSqlTaskExecution.java b/core/trino-main/src/test/java/io/trino/execution/TestSqlTaskExecution.java index 963950b7e518..bc86ed1ac435 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestSqlTaskExecution.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestSqlTaskExecution.java @@ -598,7 +598,6 @@ private TaskContext newTestingTaskContext(ScheduledExecutorService taskNotificat QueryContext queryContext = new QueryContext( new QueryId("queryid"), DataSize.of(1, MEGABYTE), - DataSize.of(2, MEGABYTE), Optional.empty(), new MemoryPool(new MemoryPoolId("test"), DataSize.of(1, GIGABYTE)), new TestingGcMonitor(), diff --git a/core/trino-main/src/test/java/io/trino/execution/TestSqlTaskManager.java b/core/trino-main/src/test/java/io/trino/execution/TestSqlTaskManager.java index 89a9c4475207..82cf54dff542 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestSqlTaskManager.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestSqlTaskManager.java @@ -51,7 +51,6 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.trino.SessionTestUtils.TEST_SESSION; import static io.trino.SystemSessionProperties.QUERY_MAX_MEMORY_PER_NODE; -import static io.trino.SystemSessionProperties.QUERY_MAX_TOTAL_MEMORY_PER_NODE; import static io.trino.execution.TaskTestUtils.PLAN_FRAGMENT; import static io.trino.execution.TaskTestUtils.SPLIT; import static io.trino.execution.TaskTestUtils.TABLE_SCAN_NODE_ID; @@ -242,8 +241,7 @@ public void testRemoveOldTasks() public void testSessionPropertyMemoryLimitOverride() { NodeMemoryConfig memoryConfig = new NodeMemoryConfig() - .setMaxQueryMemoryPerNode(DataSize.ofBytes(3)) - .setMaxQueryTotalMemoryPerNode(DataSize.ofBytes(4)); + .setMaxQueryMemoryPerNode(DataSize.ofBytes(3)); try (SqlTaskManager sqlTaskManager = createSqlTaskManager(new TaskManagerConfig(), memoryConfig)) { TaskId reduceLimitsId = new TaskId(new StageId("q1", 0), 1, 0); @@ -255,17 +253,14 @@ public void testSessionPropertyMemoryLimitOverride() // not initialized with a task update yet assertFalse(reducesLimitsContext.isMemoryLimitsInitialized()); assertEquals(reducesLimitsContext.getMaxUserMemory(), memoryConfig.getMaxQueryMemoryPerNode().toBytes()); - assertEquals(reducesLimitsContext.getMaxTotalMemory(), memoryConfig.getMaxQueryTotalMemoryPerNode().toBytes()); assertFalse(attemptsIncreaseContext.isMemoryLimitsInitialized()); assertEquals(attemptsIncreaseContext.getMaxUserMemory(), memoryConfig.getMaxQueryMemoryPerNode().toBytes()); - assertEquals(attemptsIncreaseContext.getMaxTotalMemory(), memoryConfig.getMaxQueryTotalMemoryPerNode().toBytes()); // memory limits reduced by session properties sqlTaskManager.updateTask( testSessionBuilder() .setSystemProperty(QUERY_MAX_MEMORY_PER_NODE, "1B") - .setSystemProperty(QUERY_MAX_TOTAL_MEMORY_PER_NODE, "2B") .build(), reduceLimitsId, Optional.of(PLAN_FRAGMENT), @@ -274,13 +269,11 @@ public void testSessionPropertyMemoryLimitOverride() ImmutableMap.of()); assertTrue(reducesLimitsContext.isMemoryLimitsInitialized()); assertEquals(reducesLimitsContext.getMaxUserMemory(), 1); - assertEquals(reducesLimitsContext.getMaxTotalMemory(), 2); // memory limits not increased by session properties sqlTaskManager.updateTask( testSessionBuilder() .setSystemProperty(QUERY_MAX_MEMORY_PER_NODE, "10B") - .setSystemProperty(QUERY_MAX_TOTAL_MEMORY_PER_NODE, "10B") .build(), increaseLimitsId, Optional.of(PLAN_FRAGMENT), @@ -289,7 +282,6 @@ public void testSessionPropertyMemoryLimitOverride() ImmutableMap.of()); assertTrue(attemptsIncreaseContext.isMemoryLimitsInitialized()); assertEquals(attemptsIncreaseContext.getMaxUserMemory(), memoryConfig.getMaxQueryMemoryPerNode().toBytes()); - assertEquals(attemptsIncreaseContext.getMaxTotalMemory(), memoryConfig.getMaxQueryTotalMemoryPerNode().toBytes()); } } @@ -343,7 +335,7 @@ public static class MockDirectExchangeClientSupplier implements DirectExchangeClientSupplier { @Override - public DirectExchangeClient get(LocalMemoryContext systemMemoryContext, TaskFailureListener taskFailureListener, RetryPolicy retryPolicy) + public DirectExchangeClient get(LocalMemoryContext memoryContext, TaskFailureListener taskFailureListener, RetryPolicy retryPolicy) { throw new UnsupportedOperationException(); } diff --git a/core/trino-main/src/test/java/io/trino/execution/TestStageStats.java b/core/trino-main/src/test/java/io/trino/execution/TestStageStats.java index 09d970a5baf7..d727f388bc9a 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestStageStats.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestStageStats.java @@ -46,7 +46,6 @@ public class TestStageStats 11, 12.0, - 10.0, DataSize.ofBytes(13), DataSize.ofBytes(14), DataSize.ofBytes(15), @@ -118,7 +117,6 @@ private static void assertExpectedStageStats(StageStats actual) assertEquals(actual.getCompletedDrivers(), 11); assertEquals(actual.getCumulativeUserMemory(), 12.0); - assertEquals(actual.getCumulativeSystemMemory(), 10.0); assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(13)); assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(14)); assertEquals(actual.getTotalMemoryReservation(), DataSize.ofBytes(15)); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestingRemoteTaskFactory.java b/core/trino-main/src/test/java/io/trino/execution/TestingRemoteTaskFactory.java index 31e2fe8696a4..763e773e0c7c 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestingRemoteTaskFactory.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestingRemoteTaskFactory.java @@ -173,7 +173,6 @@ public TaskStatus getTaskStatus() DataSize.of(0, BYTE), DataSize.of(0, BYTE), DataSize.of(0, BYTE), - DataSize.of(0, BYTE), 0, new Duration(0, MILLISECONDS), INITIAL_DYNAMIC_FILTERS_VERSION, diff --git a/core/trino-main/src/test/java/io/trino/memory/TestLocalMemoryManager.java b/core/trino-main/src/test/java/io/trino/memory/TestLocalMemoryManager.java index e7addff913a9..06db73e0edb5 100644 --- a/core/trino-main/src/test/java/io/trino/memory/TestLocalMemoryManager.java +++ b/core/trino-main/src/test/java/io/trino/memory/TestLocalMemoryManager.java @@ -29,8 +29,7 @@ public void testReservedMemoryPoolDisabled() { NodeMemoryConfig config = new NodeMemoryConfig() .setHeapHeadroom(DataSize.of(10, GIGABYTE)) - .setMaxQueryMemoryPerNode(DataSize.of(20, GIGABYTE)) - .setMaxQueryTotalMemoryPerNode(DataSize.of(20, GIGABYTE)); + .setMaxQueryMemoryPerNode(DataSize.of(20, GIGABYTE)); LocalMemoryManager localMemoryManager = new LocalMemoryManager(config, DataSize.of(60, GIGABYTE).toBytes()); assertFalse(localMemoryManager.getReservedPool().isPresent()); @@ -43,40 +42,24 @@ public void testReservedMemoryPoolEnabled() NodeMemoryConfig config = new NodeMemoryConfig() .setReservedPoolDisabled(false) .setHeapHeadroom(DataSize.of(10, GIGABYTE)) - .setMaxQueryMemoryPerNode(DataSize.of(20, GIGABYTE)) - .setMaxQueryTotalMemoryPerNode(DataSize.of(20, GIGABYTE)); + .setMaxQueryMemoryPerNode(DataSize.of(20, GIGABYTE)); LocalMemoryManager localMemoryManager = new LocalMemoryManager(config, DataSize.of(60, GIGABYTE).toBytes()); assertTrue(localMemoryManager.getReservedPool().isPresent()); assertEquals(localMemoryManager.getPools().size(), 2); } - @Test - public void testMaxQueryMemoryPerNodeBiggerThanQueryTotalMemoryPerNode() - { - NodeMemoryConfig config = new NodeMemoryConfig() - .setHeapHeadroom(DataSize.of(10, GIGABYTE)) - .setMaxQueryMemoryPerNode(DataSize.of(200, GIGABYTE)) - .setMaxQueryTotalMemoryPerNode(DataSize.of(20, GIGABYTE)); - - assertThatThrownBy(() -> new LocalMemoryManager(config, DataSize.of(60, GIGABYTE).toBytes())) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("Max query memory per node (query.max-memory-per-node) cannot be greater than" + - " the max query total memory per node (query.max-total-memory-per-node)"); - } - @Test public void testNotEnoughAvailableMemory() { NodeMemoryConfig config = new NodeMemoryConfig() .setHeapHeadroom(DataSize.of(10, GIGABYTE)) - .setMaxQueryMemoryPerNode(DataSize.of(20, GIGABYTE)) - .setMaxQueryTotalMemoryPerNode(DataSize.of(20, GIGABYTE)); + .setMaxQueryMemoryPerNode(DataSize.of(20, GIGABYTE)); - // 25 GB heap is not sufficient for 10 GB heap headroom and 20 GB query.max-total-memory-per-node + // 25 GB heap is not sufficient for 10 GB heap headroom and 20 GB query.max-memory-per-node assertThatThrownBy(() -> new LocalMemoryManager(config, DataSize.of(25, GIGABYTE).toBytes())) .isInstanceOf(IllegalArgumentException.class) - .hasMessageMatching("Invalid memory configuration\\. The sum of max total query memory per node .* and heap headroom .*" + + .hasMessageMatching("Invalid memory configuration\\. The sum of max query memory per node .* and heap headroom .*" + "cannot be larger than the available heap memory .*"); } } diff --git a/core/trino-main/src/test/java/io/trino/memory/TestSystemMemoryBlocking.java b/core/trino-main/src/test/java/io/trino/memory/TestMemoryBlocking.java similarity index 98% rename from core/trino-main/src/test/java/io/trino/memory/TestSystemMemoryBlocking.java rename to core/trino-main/src/test/java/io/trino/memory/TestMemoryBlocking.java index 82d187f84770..476f7a5ca8a9 100644 --- a/core/trino-main/src/test/java/io/trino/memory/TestSystemMemoryBlocking.java +++ b/core/trino-main/src/test/java/io/trino/memory/TestMemoryBlocking.java @@ -59,7 +59,7 @@ import static org.testng.Assert.assertTrue; @Test(singleThreaded = true) -public class TestSystemMemoryBlocking +public class TestMemoryBlocking { private static final QueryId QUERY_ID = new QueryId("test_query"); @@ -92,7 +92,7 @@ public void tearDown() } @Test - public void testTableScanSystemMemoryBlocking() + public void testTableScanMemoryBlocking() { PlanNodeId sourceId = new PlanNodeId("source"); List types = ImmutableList.of(VARCHAR); diff --git a/core/trino-main/src/test/java/io/trino/memory/TestMemoryPools.java b/core/trino-main/src/test/java/io/trino/memory/TestMemoryPools.java index 6db5d2867b24..536507b50236 100644 --- a/core/trino-main/src/test/java/io/trino/memory/TestMemoryPools.java +++ b/core/trino-main/src/test/java/io/trino/memory/TestMemoryPools.java @@ -96,7 +96,6 @@ private void setUp(Supplier> driversSupplier) SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(DataSize.of(1, GIGABYTE)); QueryContext queryContext = new QueryContext(new QueryId("query"), TEN_MEGABYTES, - DataSize.of(20, MEGABYTE), Optional.empty(), userPool, new TestingGcMonitor(), diff --git a/core/trino-main/src/test/java/io/trino/memory/TestMemoryTracking.java b/core/trino-main/src/test/java/io/trino/memory/TestMemoryTracking.java index 1d1de4f74eb5..5840c833fb62 100644 --- a/core/trino-main/src/test/java/io/trino/memory/TestMemoryTracking.java +++ b/core/trino-main/src/test/java/io/trino/memory/TestMemoryTracking.java @@ -58,7 +58,6 @@ public class TestMemoryTracking { private static final DataSize queryMaxMemory = DataSize.of(1, GIGABYTE); - private static final DataSize queryMaxTotalMemory = DataSize.of(1, GIGABYTE); private static final DataSize memoryPoolSize = DataSize.of(1, GIGABYTE); private static final DataSize maxSpillSize = DataSize.of(1, GIGABYTE); private static final DataSize queryMaxSpillSize = DataSize.of(1, GIGABYTE); @@ -96,16 +95,15 @@ public void tearDown() @BeforeMethod public void setUpTest() { - setupTestWithLimits(queryMaxMemory, queryMaxTotalMemory, Optional.empty()); + setupTestWithLimits(queryMaxMemory, Optional.empty()); } - private void setupTestWithLimits(DataSize queryMaxMemory, DataSize queryMaxTotalMemory, Optional queryMaxTaskMemory) + private void setupTestWithLimits(DataSize queryMaxMemory, Optional queryMaxTaskMemory) { memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize); queryContext = new QueryContext( new QueryId("test_query"), queryMaxMemory, - queryMaxTotalMemory, queryMaxTaskMemory, memoryPool, new TestingGcMonitor(), @@ -128,98 +126,88 @@ private void setupTestWithLimits(DataSize queryMaxMemory, DataSize queryMaxTotal public void testOperatorAllocations() { MemoryTrackingContext operatorMemoryContext = operatorContext.getOperatorMemoryContext(); - LocalMemoryContext systemMemory = operatorContext.newLocalSystemMemoryContext("test"); LocalMemoryContext userMemory = operatorContext.localUserMemoryContext(); LocalMemoryContext revocableMemory = operatorContext.localRevocableMemoryContext(); userMemory.setBytes(100); - assertOperatorMemoryAllocations(operatorMemoryContext, 100, 0, 0); - systemMemory.setBytes(1_000_000); - assertOperatorMemoryAllocations(operatorMemoryContext, 100, 1_000_000, 0); - systemMemory.setBytes(2_000_000); - assertOperatorMemoryAllocations(operatorMemoryContext, 100, 2_000_000, 0); + assertOperatorMemoryAllocations(operatorMemoryContext, 100, 0); + assertOperatorMemoryAllocations(operatorMemoryContext, 100, 0); userMemory.setBytes(500); - assertOperatorMemoryAllocations(operatorMemoryContext, 500, 2_000_000, 0); + assertOperatorMemoryAllocations(operatorMemoryContext, 500, 0); userMemory.setBytes(userMemory.getBytes() - 500); - assertOperatorMemoryAllocations(operatorMemoryContext, 0, 2_000_000, 0); + assertOperatorMemoryAllocations(operatorMemoryContext, 0, 0); revocableMemory.setBytes(300); - assertOperatorMemoryAllocations(operatorMemoryContext, 0, 2_000_000, 300); + assertOperatorMemoryAllocations(operatorMemoryContext, 0, 300); assertThatThrownBy(() -> userMemory.setBytes(userMemory.getBytes() - 500)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("bytes cannot be negative"); operatorContext.destroy(); - assertOperatorMemoryAllocations(operatorMemoryContext, 0, 0, 0); + assertOperatorMemoryAllocations(operatorMemoryContext, 0, 0); } @Test public void testLocalTotalMemoryLimitExceeded() { - LocalMemoryContext systemMemoryContext = operatorContext.newLocalSystemMemoryContext("test"); - systemMemoryContext.setBytes(100); - assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 100, 0); - systemMemoryContext.setBytes(queryMaxTotalMemory.toBytes()); - assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, queryMaxTotalMemory.toBytes(), 0); - assertThatThrownBy(() -> systemMemoryContext.setBytes(queryMaxTotalMemory.toBytes() + 1)) + LocalMemoryContext memoryContext = operatorContext.newLocalUserMemoryContext("test"); + memoryContext.setBytes(100); + assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 100, 0); + memoryContext.setBytes(queryMaxMemory.toBytes()); + assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), queryMaxMemory.toBytes(), 0); + assertThatThrownBy(() -> memoryContext.setBytes(queryMaxMemory.toBytes() + 1)) .isInstanceOf(ExceededMemoryLimitException.class) - .hasMessage("Query exceeded per-node total memory limit of %1$s [Allocated: %1$s, Delta: 1B, Top Consumers: {test=%1$s}]", queryMaxTotalMemory); + .hasMessage("Query exceeded per-node memory limit of %1$s [Allocated: %1$s, Delta: 1B, Top Consumers: {test=%1$s}]", queryMaxMemory); } @Test public void testTaskMemoryLimitExceeded() { DataSize taskMaxMemory = DataSize.of(1, GIGABYTE); - setupTestWithLimits(DataSize.of(2, GIGABYTE), DataSize.of(2, GIGABYTE), Optional.of(taskMaxMemory)); - LocalMemoryContext systemMemoryContext = operatorContext.newLocalSystemMemoryContext("test"); - systemMemoryContext.setBytes(100); - assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 100, 0); - systemMemoryContext.setBytes(taskMaxMemory.toBytes()); - assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, taskMaxMemory.toBytes(), 0); - assertThatThrownBy(() -> systemMemoryContext.setBytes(taskMaxMemory.toBytes() + 1)) + setupTestWithLimits(DataSize.of(2, GIGABYTE), Optional.of(taskMaxMemory)); + LocalMemoryContext memoryContext = operatorContext.newLocalUserMemoryContext("test"); + memoryContext.setBytes(100); + assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 100, 0); + memoryContext.setBytes(taskMaxMemory.toBytes()); + assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), taskMaxMemory.toBytes(), 0); + assertThatThrownBy(() -> memoryContext.setBytes(taskMaxMemory.toBytes() + 1)) .isInstanceOf(ExceededMemoryLimitException.class) - .hasMessage("Query exceeded per-task total memory limit of %1$s [Allocated: %s, Delta: 1B, Top Consumers: {test=%s}]", taskMaxMemory, DataSize.succinctBytes(taskMaxMemory.toBytes() + 1)); + .hasMessage("Query exceeded per-task memory limit of %1$s [Allocated: %s, Delta: 1B, Top Consumers: {test=%s}]", taskMaxMemory, DataSize.succinctBytes(taskMaxMemory.toBytes() + 1)); } @Test - public void testLocalSystemAllocations() + public void testLocalAllocations() { long pipelineLocalAllocation = 1_000_000; long taskLocalAllocation = 10_000_000; - LocalMemoryContext pipelineLocalSystemMemoryContext = pipelineContext.localSystemMemoryContext(); - pipelineLocalSystemMemoryContext.setBytes(pipelineLocalAllocation); + LocalMemoryContext pipelineLocalMemoryContext = pipelineContext.localMemoryContext(); + pipelineLocalMemoryContext.setBytes(pipelineLocalAllocation); assertLocalMemoryAllocations(pipelineContext.getPipelineMemoryContext(), pipelineLocalAllocation, - 0, - pipelineLocalAllocation); - LocalMemoryContext taskLocalSystemMemoryContext = taskContext.localSystemMemoryContext(); - taskLocalSystemMemoryContext.setBytes(taskLocalAllocation); + 1_000_000); + LocalMemoryContext taskLocalMemoryContext = taskContext.localMemoryContext(); + taskLocalMemoryContext.setBytes(taskLocalAllocation); assertLocalMemoryAllocations( taskContext.getTaskMemoryContext(), pipelineLocalAllocation + taskLocalAllocation, - 0, - taskLocalAllocation); - assertEquals(pipelineContext.getPipelineStats().getSystemMemoryReservation().toBytes(), + 11_000_000); + assertEquals(pipelineContext.getPipelineStats().getUserMemoryReservation().toBytes(), pipelineLocalAllocation, "task level allocations should not be visible at the pipeline level"); - pipelineLocalSystemMemoryContext.setBytes(pipelineLocalSystemMemoryContext.getBytes() - pipelineLocalAllocation); + pipelineLocalMemoryContext.setBytes(pipelineLocalMemoryContext.getBytes() - pipelineLocalAllocation); assertLocalMemoryAllocations( pipelineContext.getPipelineMemoryContext(), taskLocalAllocation, - 0, 0); - taskLocalSystemMemoryContext.setBytes(taskLocalSystemMemoryContext.getBytes() - taskLocalAllocation); + taskLocalMemoryContext.setBytes(taskLocalMemoryContext.getBytes() - taskLocalAllocation); assertLocalMemoryAllocations( taskContext.getTaskMemoryContext(), 0, - 0, 0); } @Test public void testStats() { - LocalMemoryContext systemMemory = operatorContext.newLocalSystemMemoryContext("test"); LocalMemoryContext userMemory = operatorContext.localUserMemoryContext(); userMemory.setBytes(100_000_000); - systemMemory.setBytes(200_000_000); assertStats( operatorContext.getNestedOperatorStats(), @@ -227,8 +215,7 @@ public void testStats() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 100_000_000, - 0, - 200_000_000); + 0); // allocate more and check peak memory reservation userMemory.setBytes(600_000_000); @@ -238,8 +225,7 @@ public void testStats() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 600_000_000, - 0, - 200_000_000); + 0); userMemory.setBytes(userMemory.getBytes() - 300_000_000); assertStats( @@ -248,8 +234,7 @@ public void testStats() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 300_000_000, - 0, - 200_000_000); + 0); userMemory.setBytes(userMemory.getBytes() - 300_000_000); assertStats( @@ -258,8 +243,7 @@ public void testStats() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 0, - 0, - 200_000_000); + 0); operatorContext.destroy(); @@ -269,14 +253,12 @@ public void testStats() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 0, - 0, 0); } @Test public void testRevocableMemoryAllocations() { - LocalMemoryContext systemMemory = operatorContext.newLocalSystemMemoryContext("test"); LocalMemoryContext userMemory = operatorContext.localUserMemoryContext(); LocalMemoryContext revocableMemory = operatorContext.localRevocableMemoryContext(); revocableMemory.setBytes(100_000_000); @@ -286,10 +268,8 @@ public void testRevocableMemoryAllocations() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 0, - 100_000_000, - 0); + 100_000_000); userMemory.setBytes(100_000_000); - systemMemory.setBytes(100_000_000); revocableMemory.setBytes(200_000_000); assertStats( operatorContext.getNestedOperatorStats(), @@ -297,8 +277,7 @@ public void testRevocableMemoryAllocations() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 100_000_000, - 200_000_000, - 100_000_000); + 200_000_000); } @Test @@ -312,7 +291,6 @@ public void testTrySetBytes() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 100_000_000, - 0, 0); assertTrue(localMemoryContext.trySetBytes(200_000_000)); @@ -322,7 +300,6 @@ public void testTrySetBytes() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 200_000_000, - 0, 0); assertTrue(localMemoryContext.trySetBytes(100_000_000)); @@ -332,7 +309,6 @@ public void testTrySetBytes() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 100_000_000, - 0, 0); // allocating more than the pool size should fail and we should have the same stats as before @@ -343,7 +319,6 @@ public void testTrySetBytes() pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 100_000_000, - 0, 0); } @@ -360,16 +335,13 @@ public void testTrySetZeroBytesFullPool() @Test public void testDestroy() { - LocalMemoryContext newLocalSystemMemoryContext = operatorContext.newLocalSystemMemoryContext("test"); LocalMemoryContext newLocalUserMemoryContext = operatorContext.localUserMemoryContext(); LocalMemoryContext newLocalRevocableMemoryContext = operatorContext.localRevocableMemoryContext(); - newLocalSystemMemoryContext.setBytes(100_000); newLocalRevocableMemoryContext.setBytes(200_000); newLocalUserMemoryContext.setBytes(400_000); - assertEquals(operatorContext.getOperatorMemoryContext().getSystemMemory(), 100_000); assertEquals(operatorContext.getOperatorMemoryContext().getUserMemory(), 400_000); operatorContext.destroy(); - assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 0, 0); + assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 0); } private void assertStats( @@ -378,8 +350,7 @@ private void assertStats( PipelineStats pipelineStats, TaskStats taskStats, long expectedUserMemory, - long expectedRevocableMemory, - long expectedSystemMemory) + long expectedRevocableMemory) { OperatorStats operatorStats = getOnlyElement(nestedOperatorStats); assertEquals(operatorStats.getUserMemoryReservation().toBytes(), expectedUserMemory); @@ -387,11 +358,6 @@ private void assertStats( assertEquals(pipelineStats.getUserMemoryReservation().toBytes(), expectedUserMemory); assertEquals(taskStats.getUserMemoryReservation().toBytes(), expectedUserMemory); - assertEquals(operatorStats.getSystemMemoryReservation().toBytes(), expectedSystemMemory); - assertEquals(driverStats.getSystemMemoryReservation().toBytes(), expectedSystemMemory); - assertEquals(pipelineStats.getSystemMemoryReservation().toBytes(), expectedSystemMemory); - assertEquals(taskStats.getSystemMemoryReservation().toBytes(), expectedSystemMemory); - assertEquals(operatorStats.getRevocableMemoryReservation().toBytes(), expectedRevocableMemory); assertEquals(driverStats.getRevocableMemoryReservation().toBytes(), expectedRevocableMemory); assertEquals(pipelineStats.getRevocableMemoryReservation().toBytes(), expectedRevocableMemory); @@ -402,13 +368,10 @@ private void assertStats( private void assertOperatorMemoryAllocations( MemoryTrackingContext memoryTrackingContext, long expectedUserMemory, - long expectedSystemMemory, long expectedRevocableMemory) { assertEquals(memoryTrackingContext.getUserMemory(), expectedUserMemory, "User memory verification failed"); - // both user and system memory are allocated from the same memoryPool - assertEquals(memoryPool.getReservedBytes(), expectedUserMemory + expectedSystemMemory, "Memory pool verification failed"); - assertEquals(memoryTrackingContext.getSystemMemory(), expectedSystemMemory, "System memory verification failed"); + assertEquals(memoryPool.getReservedBytes(), expectedUserMemory, "Memory pool verification failed"); assertEquals(memoryTrackingContext.getRevocableMemory(), expectedRevocableMemory, "Revocable memory verification failed"); } @@ -416,11 +379,9 @@ private void assertOperatorMemoryAllocations( private void assertLocalMemoryAllocations( MemoryTrackingContext memoryTrackingContext, long expectedPoolMemory, - long expectedContextUserMemory, - long expectedContextSystemMemory) + long expectedContextUserMemory) { assertEquals(memoryTrackingContext.getUserMemory(), expectedContextUserMemory, "User memory verification failed"); assertEquals(memoryPool.getReservedBytes(), expectedPoolMemory, "Memory pool verification failed"); - assertEquals(memoryTrackingContext.localSystemMemoryContext().getBytes(), expectedContextSystemMemory, "Local system memory verification failed"); } } diff --git a/core/trino-main/src/test/java/io/trino/memory/TestNodeMemoryConfig.java b/core/trino-main/src/test/java/io/trino/memory/TestNodeMemoryConfig.java index 2e881e3ee709..cfec17ebaae0 100644 --- a/core/trino-main/src/test/java/io/trino/memory/TestNodeMemoryConfig.java +++ b/core/trino-main/src/test/java/io/trino/memory/TestNodeMemoryConfig.java @@ -32,9 +32,8 @@ public class TestNodeMemoryConfig public void testDefaults() { assertRecordedDefaults(recordDefaults(NodeMemoryConfig.class) - .setMaxQueryMemoryPerNode(DataSize.ofBytes(Math.round(AVAILABLE_HEAP_MEMORY * 0.1))) - .setMaxQueryTotalMemoryPerNode(DataSize.ofBytes(Math.round(AVAILABLE_HEAP_MEMORY * 0.3))) - .setMaxQueryTotalMemoryPerTask(null) + .setMaxQueryMemoryPerNode(DataSize.ofBytes(Math.round(AVAILABLE_HEAP_MEMORY * 0.3))) + .setMaxQueryMemoryPerTask(null) .setHeapHeadroom(DataSize.ofBytes(Math.round(AVAILABLE_HEAP_MEMORY * 0.3))) .setReservedPoolDisabled(true)); } @@ -44,16 +43,14 @@ public void testExplicitPropertyMappings() { Map properties = new ImmutableMap.Builder() .put("query.max-memory-per-node", "1GB") - .put("query.max-total-memory-per-node", "3GB") - .put("query.max-total-memory-per-task", "200MB") + .put("query.max-memory-per-task", "200MB") .put("memory.heap-headroom-per-node", "1GB") .put("experimental.reserved-pool-disabled", "false") .buildOrThrow(); NodeMemoryConfig expected = new NodeMemoryConfig() .setMaxQueryMemoryPerNode(DataSize.of(1, GIGABYTE)) - .setMaxQueryTotalMemoryPerNode(DataSize.of(3, GIGABYTE)) - .setMaxQueryTotalMemoryPerTask(DataSize.of(200, MEGABYTE)) + .setMaxQueryMemoryPerTask(DataSize.of(200, MEGABYTE)) .setHeapHeadroom(DataSize.of(1, GIGABYTE)) .setReservedPoolDisabled(false); diff --git a/core/trino-main/src/test/java/io/trino/memory/TestQueryContext.java b/core/trino-main/src/test/java/io/trino/memory/TestQueryContext.java index a60855b78e0c..5acdbc084e02 100644 --- a/core/trino-main/src/test/java/io/trino/memory/TestQueryContext.java +++ b/core/trino-main/src/test/java/io/trino/memory/TestQueryContext.java @@ -76,7 +76,6 @@ public void testSetMemoryPool(boolean useReservedPool) QueryContext queryContext = new QueryContext( new QueryId("query"), DataSize.ofBytes(10), - DataSize.ofBytes(20), Optional.empty(), new MemoryPool(GENERAL_POOL, DataSize.ofBytes(10)), new TestingGcMonitor(), @@ -141,7 +140,6 @@ public void testMoveTaggedAllocations() private static QueryContext createQueryContext(QueryId queryId, MemoryPool generalPool) { return new QueryContext(queryId, - DataSize.ofBytes(10_000), DataSize.ofBytes(10_000), Optional.empty(), generalPool, diff --git a/core/trino-main/src/test/java/io/trino/operator/BenchmarkHashAndStreamingAggregationOperators.java b/core/trino-main/src/test/java/io/trino/operator/BenchmarkHashAndStreamingAggregationOperators.java index 9e58c3833770..0610b0cb3ab4 100644 --- a/core/trino-main/src/test/java/io/trino/operator/BenchmarkHashAndStreamingAggregationOperators.java +++ b/core/trino-main/src/test/java/io/trino/operator/BenchmarkHashAndStreamingAggregationOperators.java @@ -232,7 +232,6 @@ private OperatorFactory createStreamingAggregationOperatorFactory( hashTypes, hashTypes, hashChannels, - SINGLE, ImmutableList.of( COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), LONG_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(sumChannel), OptionalInt.empty())), @@ -267,8 +266,7 @@ private OperatorFactory createHashAggregationOperatorFactory( succinctBytes(Integer.MAX_VALUE), spillerFactory, JOIN_COMPILER, - BLOCK_TYPE_OPERATORS, - false); + BLOCK_TYPE_OPERATORS); } private static void repeatToBigintBlock(long value, int count, BlockBuilder blockBuilder) diff --git a/core/trino-main/src/test/java/io/trino/operator/GroupByHashYieldAssertion.java b/core/trino-main/src/test/java/io/trino/operator/GroupByHashYieldAssertion.java index 7112aa5d40cf..2ef903cd19e4 100644 --- a/core/trino-main/src/test/java/io/trino/operator/GroupByHashYieldAssertion.java +++ b/core/trino-main/src/test/java/io/trino/operator/GroupByHashYieldAssertion.java @@ -83,7 +83,6 @@ public static GroupByHashYieldResult finishOperatorWithYieldingGroupByHash(List< QueryContext queryContext = new QueryContext( queryId, DataSize.of(512, MEGABYTE), - DataSize.of(1024, MEGABYTE), Optional.empty(), memoryPool, new TestingGcMonitor(), diff --git a/core/trino-main/src/test/java/io/trino/operator/TestAggregationOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestAggregationOperator.java index 1c25ad962bee..3cf73b2302d0 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestAggregationOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestAggregationOperator.java @@ -100,8 +100,7 @@ public void testMaskWithDirtyNulls() OperatorFactory operatorFactory = new AggregationOperatorFactory( 0, new PlanNodeId("test"), - ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.of(1))), - false); + ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.of(1)))); DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION) .addPipelineContext(0, true, true, false) @@ -126,8 +125,7 @@ public void testDistinctMaskWithNulls() OperatorFactory operatorFactory = new AggregationOperatorFactory( 0, new PlanNodeId("test"), - ImmutableList.of(distinctFactory), - false); + ImmutableList.of(distinctFactory)); ByteArrayBlock trueMaskAllNull = new ByteArrayBlock( 4, @@ -168,8 +166,7 @@ public void testAggregation() LONG_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(3), OptionalInt.empty()), REAL_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(4), OptionalInt.empty()), DOUBLE_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(5), OptionalInt.empty()), - maxVarcharColumn.createAggregatorFactory(SINGLE, ImmutableList.of(6), OptionalInt.empty())), - false); + maxVarcharColumn.createAggregatorFactory(SINGLE, ImmutableList.of(6), OptionalInt.empty()))); DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION) .addPipelineContext(0, true, true, false) @@ -180,28 +177,19 @@ public void testAggregation() .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); - assertEquals(driverContext.getSystemMemoryUsage(), 0); assertEquals(driverContext.getMemoryUsage(), 0); } @Test public void testMemoryTracking() throws Exception - { - testMemoryTracking(false); - testMemoryTracking(true); - } - - private void testMemoryTracking(boolean useSystemMemory) - throws Exception { Page input = getOnlyElement(rowPagesBuilder(BIGINT).addSequencePage(100, 0).build()); OperatorFactory operatorFactory = new AggregationOperatorFactory( 0, new PlanNodeId("test"), - ImmutableList.of(LONG_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty())), - useSystemMemory); + ImmutableList.of(LONG_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()))); DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION) .addPipelineContext(0, true, true, false) @@ -211,19 +199,11 @@ private void testMemoryTracking(boolean useSystemMemory) assertTrue(operator.needsInput()); operator.addInput(input); - if (useSystemMemory) { - assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0); - assertEquals(driverContext.getMemoryUsage(), 0); - } - else { - assertEquals(driverContext.getSystemMemoryUsage(), 0); - assertThat(driverContext.getMemoryUsage()).isGreaterThan(0); - } + assertThat(driverContext.getMemoryUsage()).isGreaterThan(0); toPages(operator, emptyIterator()); } - assertEquals(driverContext.getSystemMemoryUsage(), 0); assertEquals(driverContext.getMemoryUsage(), 0); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/TestDriverStats.java b/core/trino-main/src/test/java/io/trino/operator/TestDriverStats.java index 2800edc43606..a22939532601 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestDriverStats.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestDriverStats.java @@ -41,7 +41,6 @@ public class TestDriverStats DataSize.ofBytes(6), DataSize.ofBytes(7), - DataSize.ofBytes(8), new Duration(9, NANOSECONDS), new Duration(10, NANOSECONDS), @@ -94,7 +93,6 @@ public static void assertExpectedDriverStats(DriverStats actual) assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(6)); assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(7)); - assertEquals(actual.getSystemMemoryReservation(), DataSize.ofBytes(8)); assertEquals(actual.getTotalScheduledTime(), new Duration(9, NANOSECONDS)); assertEquals(actual.getTotalCpuTime(), new Duration(10, NANOSECONDS)); diff --git a/core/trino-main/src/test/java/io/trino/operator/TestExchangeOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestExchangeOperator.java index e4f95b774a1c..dc0a88f2a8c5 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestExchangeOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestExchangeOperator.java @@ -91,7 +91,7 @@ public void setUp() pageBufferClientCallbackExecutor = Executors.newSingleThreadExecutor(); httpClient = new TestingHttpClient(new TestingExchangeHttpClientHandler(taskBuffers), scheduler); - directExchangeClientSupplier = (systemMemoryUsageListener, taskFailureListener, retryPolicy) -> new DirectExchangeClient( + directExchangeClientSupplier = (memoryContext, taskFailureListener, retryPolicy) -> new DirectExchangeClient( "localhost", DataIntegrityVerification.ABORT, new StreamingDirectExchangeBuffer(scheduler, DataSize.of(32, MEGABYTE)), @@ -101,7 +101,7 @@ public void setUp() true, httpClient, scheduler, - systemMemoryUsageListener, + memoryContext, pageBufferClientCallbackExecutor, taskFailureListener); } @@ -266,7 +266,7 @@ private SourceOperator createExchangeOperator() .addDriverContext(); SourceOperator operator = operatorFactory.createOperator(driverContext); - assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getSystemMemoryReservation().toBytes(), 0); + assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getUserMemoryReservation().toBytes(), 0); return operator; } @@ -283,7 +283,7 @@ private static List waitForPages(Operator operator, int expectedPageCount) break; } - if (operator.getOperatorContext().getDriverContext().getPipelineContext().getPipelineStats().getSystemMemoryReservation().toBytes() > 0) { + if (operator.getOperatorContext().getDriverContext().getPipelineContext().getPipelineStats().getUserMemoryReservation().toBytes() > 0) { greaterThanZero = true; break; } @@ -321,7 +321,7 @@ private static List waitForPages(Operator operator, int expectedPageCount) assertPageEquals(TYPES, page, PAGE); } - assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getSystemMemoryReservation().toBytes(), 0); + assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getUserMemoryReservation().toBytes(), 0); return outputPages; } @@ -344,6 +344,6 @@ private static void waitForFinished(Operator operator) assertEquals(operator.isFinished(), true); assertEquals(operator.needsInput(), false); assertNull(operator.getOutput()); - assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getSystemMemoryReservation().toBytes(), 0); + assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getUserMemoryReservation().toBytes(), 0); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/TestHashAggregationOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestHashAggregationOperator.java index 343c64d52052..18d4ada2edea 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestHashAggregationOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestHashAggregationOperator.java @@ -193,8 +193,7 @@ public void testHashAggregation(boolean hashEnabled, boolean spillEnabled, boole succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); DriverContext driverContext = createDriverContext(memoryLimitForMerge); @@ -247,8 +246,7 @@ public void testHashAggregationWithGlobals(boolean hashEnabled, boolean spillEna succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); DriverContext driverContext = createDriverContext(memoryLimitForMerge); MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT) @@ -272,7 +270,7 @@ public void testHashAggregationMemoryReservation(boolean hashEnabled, boolean sp .addSequencePage(10, 300, 0) .build(); - DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, DataSize.of(10, Unit.MEGABYTE)) + DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, DataSize.of(11, Unit.MEGABYTE)) .addPipelineContext(0, true, true, false) .addDriverContext(); @@ -294,18 +292,16 @@ public void testHashAggregationMemoryReservation(boolean hashEnabled, boolean sp succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); Operator operator = operatorFactory.createOperator(driverContext); toPages(operator, input.iterator(), revokeMemoryWhenAddingPages); - assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getUserMemoryReservation().toBytes(), 0); - assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getRevocableMemoryReservation().toBytes(), 0); // TODO (https://github.com/trinodb/trino/issues/10596): it should be 0, since operator is finished - assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getSystemMemoryReservation().toBytes(), spillEnabled && revokeMemoryWhenAddingPages ? 5_350_968 : 0); + assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getUserMemoryReservation().toBytes(), spillEnabled && revokeMemoryWhenAddingPages ? 5_350_968 : 0); + assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getRevocableMemoryReservation().toBytes(), 0); } - @Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of 10B.*") + @Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node memory limit of 10B.*") public void testMemoryLimit(boolean hashEnabled) { TestingAggregationFunction maxVarcharColumn = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("max"), fromTypes(VARCHAR)); @@ -338,8 +334,7 @@ public void testMemoryLimit(boolean hashEnabled) 100_000, Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); toPages(operatorFactory, driverContext, input); } @@ -379,8 +374,7 @@ public void testHashBuilderResize(boolean hashEnabled, boolean spillEnabled, boo succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); toPages(operatorFactory, driverContext, input, revokeMemoryWhenAddingPages); } @@ -402,8 +396,7 @@ public void testMemoryReservationYield(Type type) 1, Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); // get result with yield; pick a relatively small buffer for aggregator's memory usage GroupByHashYieldResult result; @@ -423,7 +416,7 @@ public void testMemoryReservationYield(Type type) assertEquals(count, 6_000 * 600); } - @Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of 3MB.*") + @Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node memory limit of 3MB.*") public void testHashBuilderResizeLimit(boolean hashEnabled) { BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1, MAX_BLOCK_SIZE_IN_BYTES); @@ -455,8 +448,7 @@ public void testHashBuilderResizeLimit(boolean hashEnabled) 100_000, Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); toPages(operatorFactory, driverContext, input); } @@ -490,8 +482,7 @@ public void testMultiSliceAggregationOutput(boolean hashEnabled) 100_000, Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); assertEquals(toPages(operatorFactory, createDriverContext(), input).size(), 2); } @@ -522,8 +513,7 @@ public void testMultiplePartialFlushes(boolean hashEnabled) 100_000, Optional.of(DataSize.of(1, KILOBYTE)), joinCompiler, - blockTypeOperators, - true); + blockTypeOperators); DriverContext driverContext = createDriverContext(1024); @@ -542,8 +532,7 @@ public void testMultiplePartialFlushes(boolean hashEnabled) operator.addInput(inputIterator.next()); } - assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0); - assertEquals(driverContext.getMemoryUsage(), 0); + assertThat(driverContext.getMemoryUsage()).isGreaterThan(0); // Drain the output (partial flush) List outputPages = new ArrayList<>(); @@ -575,7 +564,6 @@ public void testMultiplePartialFlushes(boolean hashEnabled) assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); } - assertEquals(driverContext.getSystemMemoryUsage(), 0); assertEquals(driverContext.getMemoryUsage(), 0); assertEquals(driverContext.getRevocableMemoryUsage(), 0); } @@ -610,8 +598,7 @@ public void testMergeWithMemorySpill() succinctBytes(Integer.MAX_VALUE), spillerFactory, joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); DriverContext driverContext = createDriverContext(smallPagesSpillThresholdSize); @@ -665,8 +652,7 @@ public void testSpillerFailure() succinctBytes(Integer.MAX_VALUE), new FailingSpillerFactory(), joinCompiler, - blockTypeOperators, - false); + blockTypeOperators); assertThatThrownBy(() -> toPages(operatorFactory, driverContext, input)) .isInstanceOf(RuntimeException.class) @@ -677,13 +663,6 @@ public void testSpillerFailure() @Test public void testMemoryTracking() throws Exception - { - testMemoryTracking(false); - testMemoryTracking(true); - } - - private void testMemoryTracking(boolean useSystemMemory) - throws Exception { List hashChannels = Ints.asList(0); RowPagesBuilder rowPagesBuilder = rowPagesBuilder(false, hashChannels, BIGINT); @@ -702,8 +681,7 @@ private void testMemoryTracking(boolean useSystemMemory) 100_000, Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, - blockTypeOperators, - useSystemMemory); + blockTypeOperators); DriverContext driverContext = createDriverContext(1024); @@ -711,19 +689,11 @@ private void testMemoryTracking(boolean useSystemMemory) assertTrue(operator.needsInput()); operator.addInput(input); - if (useSystemMemory) { - assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0); - assertEquals(driverContext.getMemoryUsage(), 0); - } - else { - assertEquals(driverContext.getSystemMemoryUsage(), 0); - assertThat(driverContext.getMemoryUsage()).isGreaterThan(0); - } + assertThat(driverContext.getMemoryUsage()).isGreaterThan(0); toPages(operator, emptyIterator()); } - assertEquals(driverContext.getSystemMemoryUsage(), 0); assertEquals(driverContext.getMemoryUsage(), 0); assertEquals(driverContext.getRevocableMemoryUsage(), 0); } diff --git a/core/trino-main/src/test/java/io/trino/operator/TestHashSemiJoinOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestHashSemiJoinOperator.java index e3d2c72174c3..b124080c2280 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestHashSemiJoinOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestHashSemiJoinOperator.java @@ -429,7 +429,7 @@ public void testProbeAndBuildNulls(boolean hashEnabled) OperatorAssertion.assertOperatorEquals(joinOperatorFactory, driverContext, probeInput, expected, hashEnabled, ImmutableList.of(probeTypes.size())); } - @Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of.*") + @Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node memory limit of.*") public void testMemoryLimit(boolean hashEnabled) { DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, DataSize.ofBytes(100)) diff --git a/core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java b/core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java index bd48778a6802..c5ab25564946 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java @@ -76,9 +76,7 @@ public class TestOperatorStats DataSize.ofBytes(19), DataSize.ofBytes(20), - DataSize.ofBytes(21), DataSize.ofBytes(22), - DataSize.ofBytes(23), DataSize.ofBytes(24), DataSize.ofBytes(25), DataSize.ofBytes(26), @@ -125,9 +123,7 @@ public class TestOperatorStats DataSize.ofBytes(19), DataSize.ofBytes(20), - DataSize.ofBytes(21), DataSize.ofBytes(22), - DataSize.ofBytes(23), DataSize.ofBytes(24), DataSize.ofBytes(25), DataSize.ofBytes(26), @@ -184,9 +180,7 @@ public static void assertExpectedOperatorStats(OperatorStats actual) assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(19)); assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(20)); - assertEquals(actual.getSystemMemoryReservation(), DataSize.ofBytes(21)); assertEquals(actual.getPeakUserMemoryReservation(), DataSize.ofBytes(22)); - assertEquals(actual.getPeakSystemMemoryReservation(), DataSize.ofBytes(23)); assertEquals(actual.getPeakRevocableMemoryReservation(), DataSize.ofBytes(24)); assertEquals(actual.getPeakTotalMemoryReservation(), DataSize.ofBytes(25)); assertEquals(actual.getSpilledDataSize(), DataSize.ofBytes(26)); @@ -235,9 +229,7 @@ public void testAdd() assertEquals(actual.getFinishCpu(), new Duration(3 * 18, NANOSECONDS)); assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(3 * 19)); assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(3 * 20)); - assertEquals(actual.getSystemMemoryReservation(), DataSize.ofBytes(3 * 21)); assertEquals(actual.getPeakUserMemoryReservation(), DataSize.ofBytes(22)); - assertEquals(actual.getPeakSystemMemoryReservation(), DataSize.ofBytes(23)); assertEquals(actual.getPeakRevocableMemoryReservation(), DataSize.ofBytes(24)); assertEquals(actual.getPeakTotalMemoryReservation(), DataSize.ofBytes(25)); assertEquals(actual.getSpilledDataSize(), DataSize.ofBytes(3 * 26)); @@ -285,9 +277,7 @@ public void testAddMergeable() assertEquals(actual.getFinishCpu(), new Duration(3 * 18, NANOSECONDS)); assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(3 * 19)); assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(3 * 20)); - assertEquals(actual.getSystemMemoryReservation(), DataSize.ofBytes(3 * 21)); assertEquals(actual.getPeakUserMemoryReservation(), DataSize.ofBytes(22)); - assertEquals(actual.getPeakSystemMemoryReservation(), DataSize.ofBytes(23)); assertEquals(actual.getPeakRevocableMemoryReservation(), DataSize.ofBytes(24)); assertEquals(actual.getPeakTotalMemoryReservation(), DataSize.ofBytes(25)); assertEquals(actual.getSpilledDataSize(), DataSize.ofBytes(3 * 26)); diff --git a/core/trino-main/src/test/java/io/trino/operator/TestOrderByOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestOrderByOperator.java index afdfcee2478c..f00b0fc5fa15 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestOrderByOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestOrderByOperator.java @@ -263,7 +263,7 @@ public void testMemoryLimit() assertThatThrownBy(() -> toPages(operatorFactory, driverContext, input)) .isInstanceOf(ExceededMemoryLimitException.class) - .hasMessageMatching("Query exceeded per-node user memory limit of 10B.*"); + .hasMessageMatching("Query exceeded per-node memory limit of 10B.*"); } private DriverContext createDriverContext(long memoryLimit) diff --git a/core/trino-main/src/test/java/io/trino/operator/TestPipelineStats.java b/core/trino-main/src/test/java/io/trino/operator/TestPipelineStats.java index d2e4c900bca7..c93259e3b91f 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestPipelineStats.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestPipelineStats.java @@ -53,7 +53,6 @@ public class TestPipelineStats DataSize.ofBytes(5), DataSize.ofBytes(6), - DataSize.ofBytes(7), getTestDistribution(8), getTestDistribution(9), @@ -116,7 +115,6 @@ public static void assertExpectedPipelineStats(PipelineStats actual) assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(5)); assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(6)); - assertEquals(actual.getSystemMemoryReservation(), DataSize.ofBytes(7)); assertEquals(actual.getQueuedTime().getCount(), 8.0); assertEquals(actual.getElapsedTime().getCount(), 9.0); diff --git a/core/trino-main/src/test/java/io/trino/operator/TestScanFilterAndProjectOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestScanFilterAndProjectOperator.java index a3cadbf950d3..90ffd3dc0e68 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestScanFilterAndProjectOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestScanFilterAndProjectOperator.java @@ -436,7 +436,7 @@ public long getReadTimeNanos() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/core/trino-main/src/test/java/io/trino/operator/TestStreamingAggregationOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestStreamingAggregationOperator.java index ab3265973539..734891d09c42 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestStreamingAggregationOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestStreamingAggregationOperator.java @@ -75,7 +75,6 @@ public void setUp() ImmutableList.of(BOOLEAN, VARCHAR, BIGINT), ImmutableList.of(VARCHAR), ImmutableList.of(1), - SINGLE, ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), LONG_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(2), OptionalInt.empty())), new JoinCompiler(new TypeOperators())); @@ -97,7 +96,6 @@ public void test() ImmutableList.of(BOOLEAN, DOUBLE, BIGINT), ImmutableList.of(DOUBLE), ImmutableList.of(1), - SINGLE, ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), LONG_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(2), OptionalInt.empty())), new JoinCompiler(new TypeOperators())); diff --git a/core/trino-main/src/test/java/io/trino/operator/TestTableFinishOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestTableFinishOperator.java index a7e626c1ecd5..251d393430b1 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestTableFinishOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestTableFinishOperator.java @@ -103,8 +103,7 @@ public void testStatisticsAggregation() new AggregationOperator.AggregationOperatorFactory( 1, new PlanNodeId("test"), - ImmutableList.of(LONG_MAX.createAggregatorFactory(SINGLE, ImmutableList.of(2), OptionalInt.empty())), - true), + ImmutableList.of(LONG_MAX.createAggregatorFactory(SINGLE, ImmutableList.of(2), OptionalInt.empty()))), descriptor, tableExecuteContextManager, true, @@ -124,8 +123,7 @@ public void testStatisticsAggregation() operator.addInput(rowPagesBuilder(inputTypes).row(null, null, 6).build().get(0)); operator.addInput(rowPagesBuilder(inputTypes).row(null, null, 7).build().get(0)); - assertThat(driverContext.getSystemMemoryUsage()).as("systemMemoryUsage").isGreaterThan(0); - assertEquals(driverContext.getMemoryUsage(), 0, "memoryUsage"); + assertThat(driverContext.getMemoryUsage()).as("memoryUsage").isGreaterThan(0); assertTrue(operator.isBlocked().isDone(), "isBlocked should be done"); assertTrue(operator.needsInput(), "needsInput should be true"); @@ -152,7 +150,6 @@ public void testStatisticsAggregation() .build(); assertBlockEquals(BIGINT, getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().get(statisticMetadata), expectedStatisticsBlock); - assertEquals(driverContext.getSystemMemoryUsage(), 0, "systemMemoryUsage"); assertEquals(driverContext.getMemoryUsage(), 0, "memoryUsage"); } diff --git a/core/trino-main/src/test/java/io/trino/operator/TestTableWriterOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestTableWriterOperator.java index c9d9676e59b6..8db7a99eb719 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestTableWriterOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestTableWriterOperator.java @@ -208,8 +208,7 @@ public void testStatisticsAggregation() new AggregationOperatorFactory( 1, new PlanNodeId("test"), - ImmutableList.of(LONG_MAX.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty())), - true), + ImmutableList.of(LONG_MAX.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()))), outputTypes, session, driverContext); @@ -220,8 +219,7 @@ public void testStatisticsAggregation() assertTrue(operator.isBlocked().isDone()); assertTrue(operator.needsInput()); - assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0); - assertEquals(driverContext.getMemoryUsage(), 0); + assertThat(driverContext.getMemoryUsage()).isGreaterThan(0); operator.finish(); assertFalse(operator.isFinished()); @@ -250,7 +248,6 @@ private void assertMemoryIsReleased(TableWriterOperator tableWriterOperator) { OperatorContext tableWriterOperatorOperatorContext = tableWriterOperator.getOperatorContext(); MemoryTrackingContext tableWriterMemoryContext = tableWriterOperatorOperatorContext.getOperatorMemoryContext(); - assertEquals(tableWriterMemoryContext.getSystemMemory(), 0); assertEquals(tableWriterMemoryContext.getUserMemory(), 0); assertEquals(tableWriterMemoryContext.getRevocableMemory(), 0); @@ -259,7 +256,6 @@ private void assertMemoryIsReleased(TableWriterOperator tableWriterOperator) AggregationOperator aggregationOperator = (AggregationOperator) statisticAggregationOperator; OperatorContext aggregationOperatorOperatorContext = aggregationOperator.getOperatorContext(); MemoryTrackingContext aggregationOperatorMemoryContext = aggregationOperatorOperatorContext.getOperatorMemoryContext(); - assertEquals(aggregationOperatorMemoryContext.getSystemMemory(), 0); assertEquals(aggregationOperatorMemoryContext.getUserMemory(), 0); assertEquals(aggregationOperatorMemoryContext.getRevocableMemory(), 0); } @@ -384,7 +380,7 @@ public CompletableFuture> finish() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { long memoryUsage = 0; for (Page page : pages) { diff --git a/core/trino-main/src/test/java/io/trino/operator/TestTaskStats.java b/core/trino-main/src/test/java/io/trino/operator/TestTaskStats.java index 59184d013401..1c8efc9aa5c9 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestTaskStats.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestTaskStats.java @@ -48,10 +48,8 @@ public class TestTaskStats 10, 11.0, - 5.0, DataSize.ofBytes(12), DataSize.ofBytes(13), - DataSize.ofBytes(14), new Duration(15, NANOSECONDS), new Duration(16, NANOSECONDS), new Duration(18, NANOSECONDS), @@ -113,10 +111,8 @@ public static void assertExpectedTaskStats(TaskStats actual) assertEquals(actual.getCompletedDrivers(), 10); assertEquals(actual.getCumulativeUserMemory(), 11.0); - assertEquals(actual.getCumulativeSystemMemory(), 5.0); assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(12)); assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(13)); - assertEquals(actual.getSystemMemoryReservation(), DataSize.ofBytes(14)); assertEquals(actual.getTotalScheduledTime(), new Duration(15, NANOSECONDS)); assertEquals(actual.getTotalCpuTime(), new Duration(16, NANOSECONDS)); diff --git a/core/trino-main/src/test/java/io/trino/operator/TestTopNOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestTopNOperator.java index f86a59462ba4..dfef2508c3cd 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestTopNOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestTopNOperator.java @@ -201,7 +201,7 @@ public void testExceedMemoryLimit() Operator operator = operatorFactory.createOperator(smallDiverContext); assertThatThrownBy(() -> operator.addInput(input.get(0))) .isInstanceOf(ExceededMemoryLimitException.class) - .hasMessageStartingWith("Query exceeded per-node user memory limit of "); + .hasMessageStartingWith("Query exceeded per-node memory limit of "); } private OperatorFactory topNOperatorFactory( diff --git a/core/trino-main/src/test/java/io/trino/operator/TestWindowOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestWindowOperator.java index 11982b8833d3..534c43daa604 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestWindowOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestWindowOperator.java @@ -369,7 +369,7 @@ public void testDistinctPartitionAndPeers(boolean spillEnabled, boolean revokeMe assertOperatorEquals(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages); } - @Test(expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of 10B.*") + @Test(expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node memory limit of 10B.*") public void testMemoryLimit() { List input = rowPagesBuilder(BIGINT, DOUBLE) diff --git a/core/trino-main/src/test/java/io/trino/operator/TestingOperatorContext.java b/core/trino-main/src/test/java/io/trino/operator/TestingOperatorContext.java index b147efadd27d..73a89d0ea2fe 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestingOperatorContext.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestingOperatorContext.java @@ -36,7 +36,7 @@ public static OperatorContext create(ScheduledExecutorService scheduledExecutor) scheduledExecutor, TestingSession.testSessionBuilder().build()); - MemoryTrackingContext pipelineMemoryContext = new MemoryTrackingContext(newSimpleAggregatedMemoryContext(), newSimpleAggregatedMemoryContext(), newSimpleAggregatedMemoryContext()); + MemoryTrackingContext pipelineMemoryContext = new MemoryTrackingContext(newSimpleAggregatedMemoryContext(), newSimpleAggregatedMemoryContext()); PipelineContext pipelineContext = new PipelineContext( 1, diff --git a/core/trino-main/src/test/java/io/trino/operator/join/TestHashJoinOperator.java b/core/trino-main/src/test/java/io/trino/operator/join/TestHashJoinOperator.java index cc8865aad78b..1fc92ee44dd5 100644 --- a/core/trino-main/src/test/java/io/trino/operator/join/TestHashJoinOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/join/TestHashJoinOperator.java @@ -1214,7 +1214,7 @@ public void testMemoryLimit(boolean parallelBuild, boolean buildHashEnabled) assertThatThrownBy(() -> buildLookupSource(executor, buildSideSetup)) .isInstanceOf(ExceededMemoryLimitException.class) - .hasMessageMatching("Query exceeded per-node user memory limit of.*"); + .hasMessageMatching("Query exceeded per-node memory limit of.*"); } @Test(dataProvider = "hashJoinTestValues") diff --git a/core/trino-main/src/test/java/io/trino/operator/output/BenchmarkPartitionedOutputOperator.java b/core/trino-main/src/test/java/io/trino/operator/output/BenchmarkPartitionedOutputOperator.java index 3c1a96e50481..a8c78741b5ef 100644 --- a/core/trino-main/src/test/java/io/trino/operator/output/BenchmarkPartitionedOutputOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/output/BenchmarkPartitionedOutputOperator.java @@ -479,11 +479,11 @@ public TestingPartitionedOutputBuffer( StateMachine state, OutputBuffers outputBuffers, DataSize maxBufferSize, - Supplier systemMemoryContextSupplier, + Supplier memoryContextSupplier, Executor notificationExecutor, Blackhole blackhole) { - super(taskInstanceId, state, outputBuffers, maxBufferSize, systemMemoryContextSupplier, notificationExecutor); + super(taskInstanceId, state, outputBuffers, maxBufferSize, memoryContextSupplier, notificationExecutor); this.blackhole = blackhole; } diff --git a/core/trino-main/src/test/java/io/trino/server/TestBasicQueryInfo.java b/core/trino-main/src/test/java/io/trino/server/TestBasicQueryInfo.java index 4cfd16ddb804..1ba32d7fc31b 100644 --- a/core/trino-main/src/test/java/io/trino/server/TestBasicQueryInfo.java +++ b/core/trino-main/src/test/java/io/trino/server/TestBasicQueryInfo.java @@ -78,13 +78,11 @@ public void testConstructor() 34, 19, 20.0, - 10.0, DataSize.valueOf("21GB"), DataSize.valueOf("22GB"), DataSize.valueOf("23GB"), DataSize.valueOf("24GB"), DataSize.valueOf("25GB"), - DataSize.valueOf("30GB"), DataSize.valueOf("26GB"), DataSize.valueOf("27GB"), DataSize.valueOf("28GB"), @@ -158,7 +156,6 @@ public void testConstructor() assertEquals(basicInfo.getQueryStats().getCompletedDrivers(), 19); assertEquals(basicInfo.getQueryStats().getCumulativeUserMemory(), 20.0); - assertEquals(basicInfo.getQueryStats().getCumulativeSystemMemory(), 10.0); assertEquals(basicInfo.getQueryStats().getUserMemoryReservation(), DataSize.valueOf("21GB")); assertEquals(basicInfo.getQueryStats().getTotalMemoryReservation(), DataSize.valueOf("23GB")); assertEquals(basicInfo.getQueryStats().getPeakUserMemoryReservation(), DataSize.valueOf("24GB")); diff --git a/core/trino-main/src/test/java/io/trino/server/TestQueryStateInfo.java b/core/trino-main/src/test/java/io/trino/server/TestQueryStateInfo.java index 4ba5ecb1b5dd..1f1986bf3875 100644 --- a/core/trino-main/src/test/java/io/trino/server/TestQueryStateInfo.java +++ b/core/trino-main/src/test/java/io/trino/server/TestQueryStateInfo.java @@ -129,13 +129,11 @@ private QueryInfo createQueryInfo(String queryId, QueryState state, String query 34, 19, 20.0, - 10.0, DataSize.valueOf("21GB"), DataSize.valueOf("22GB"), DataSize.valueOf("23GB"), DataSize.valueOf("24GB"), DataSize.valueOf("25GB"), - DataSize.valueOf("30GB"), DataSize.valueOf("26GB"), DataSize.valueOf("27GB"), DataSize.valueOf("28GB"), diff --git a/core/trino-main/src/test/java/io/trino/server/remotetask/TestHttpRemoteTask.java b/core/trino-main/src/test/java/io/trino/server/remotetask/TestHttpRemoteTask.java index ea04c5f6a1f1..d60a585f6404 100644 --- a/core/trino-main/src/test/java/io/trino/server/remotetask/TestHttpRemoteTask.java +++ b/core/trino-main/src/test/java/io/trino/server/remotetask/TestHttpRemoteTask.java @@ -767,7 +767,6 @@ private TaskStatus buildTaskStatus() initialTaskStatus.isOutputBufferOverutilized(), initialTaskStatus.getPhysicalWrittenDataSize(), initialTaskStatus.getMemoryReservation(), - initialTaskStatus.getSystemMemoryReservation(), initialTaskStatus.getRevocableMemoryReservation(), initialTaskStatus.getFullGcCount(), initialTaskStatus.getFullGcTime(), diff --git a/core/trino-main/src/test/java/io/trino/spiller/TestGenericPartitioningSpiller.java b/core/trino-main/src/test/java/io/trino/spiller/TestGenericPartitioningSpiller.java index f6c5551fb556..c16fcbac77ca 100644 --- a/core/trino-main/src/test/java/io/trino/spiller/TestGenericPartitioningSpiller.java +++ b/core/trino-main/src/test/java/io/trino/spiller/TestGenericPartitioningSpiller.java @@ -215,8 +215,8 @@ private void assertSpilledPages( private static AggregatedMemoryContext mockMemoryContext(ScheduledExecutorService scheduledExecutor) { - // It's important to use OperatorContext's system memory context, because it does additional bookkeeping. - return TestingOperatorContext.create(scheduledExecutor).newAggregateSystemMemoryContext(); + // It's important to use OperatorContext's memory context, because it does additional bookkeeping. + return TestingOperatorContext.create(scheduledExecutor).newAggregateUserMemoryContext(); } private static SpillContext mockSpillContext() diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorPageSink.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorPageSink.java index 82e296e5ad7e..82cadab320aa 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorPageSink.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorPageSink.java @@ -38,7 +38,7 @@ default long getCompletedBytes() * * @return the memory used so far in table read */ - default long getSystemMemoryUsage() + default long getMemoryUsage() { return 0; } diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorPageSource.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorPageSource.java index 995562cb6c22..b316151d7a60 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorPageSource.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorPageSource.java @@ -64,7 +64,7 @@ default OptionalLong getCompletedPositions() * * @return the memory used so far in table read */ - long getSystemMemoryUsage(); + long getMemoryUsage(); /** * Immediately finishes this page source. Trino will always call this method. diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/EmptyPageSource.java b/core/trino-spi/src/main/java/io/trino/spi/connector/EmptyPageSource.java index 42945a6e8748..0cf370931c03 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/EmptyPageSource.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/EmptyPageSource.java @@ -70,7 +70,7 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/FixedPageSource.java b/core/trino-spi/src/main/java/io/trino/spi/connector/FixedPageSource.java index f54d0032dcb1..e0a548851d28 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/FixedPageSource.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/FixedPageSource.java @@ -75,7 +75,7 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return memoryUsageBytes; } diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/RecordCursor.java b/core/trino-spi/src/main/java/io/trino/spi/connector/RecordCursor.java index a3a2a7337767..2565c39c4cfc 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/RecordCursor.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/RecordCursor.java @@ -47,7 +47,7 @@ public interface RecordCursor boolean isNull(int field); - default long getSystemMemoryUsage() + default long getMemoryUsage() { // TODO: implement this method in subclasses and remove this default implementation return 0; diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/RecordPageSource.java b/core/trino-spi/src/main/java/io/trino/spi/connector/RecordPageSource.java index d101cb7114f2..dab228bf1244 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/RecordPageSource.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/RecordPageSource.java @@ -62,9 +62,9 @@ public long getReadTimeNanos() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return cursor.getSystemMemoryUsage() + pageBuilder.getSizeInBytes(); + return cursor.getMemoryUsage() + pageBuilder.getSizeInBytes(); } @Override diff --git a/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java index e2f31e69adaa..aaa3d393df63 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java +++ b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java @@ -37,8 +37,6 @@ public class QueryStatistics private final Optional executionTime; private final long peakUserMemoryBytes; - // peak of user + system memory - private final long peakTotalNonRevocableMemoryBytes; private final long peakTaskUserMemory; private final long peakTaskTotalMemory; private final long physicalInputBytes; @@ -53,7 +51,6 @@ public class QueryStatistics private final long writtenRows; private final double cumulativeMemory; - private final double cumulativeSystemMemory; private final List stageGcStatistics; @@ -84,7 +81,6 @@ public QueryStatistics( Optional planningTime, Optional executionTime, long peakUserMemoryBytes, - long peakTotalNonRevocableMemoryBytes, long peakTaskUserMemory, long peakTaskTotalMemory, long physicalInputBytes, @@ -98,7 +94,6 @@ public QueryStatistics( long writtenBytes, long writtenRows, double cumulativeMemory, - double cumulativeSystemMemory, List stageGcStatistics, int completedSplits, boolean complete, @@ -115,7 +110,6 @@ public QueryStatistics( this.planningTime = requireNonNull(planningTime, "planningTime is null"); this.executionTime = requireNonNull(executionTime, "executionTime is null"); this.peakUserMemoryBytes = peakUserMemoryBytes; - this.peakTotalNonRevocableMemoryBytes = peakTotalNonRevocableMemoryBytes; this.peakTaskUserMemory = peakTaskUserMemory; this.peakTaskTotalMemory = peakTaskTotalMemory; this.physicalInputBytes = physicalInputBytes; @@ -129,7 +123,6 @@ public QueryStatistics( this.writtenBytes = writtenBytes; this.writtenRows = writtenRows; this.cumulativeMemory = cumulativeMemory; - this.cumulativeSystemMemory = cumulativeSystemMemory; this.stageGcStatistics = requireNonNull(stageGcStatistics, "stageGcStatistics is null"); this.completedSplits = completedSplits; this.complete = complete; @@ -192,12 +185,6 @@ public long getPeakUserMemoryBytes() return peakUserMemoryBytes; } - @JsonProperty - public long getPeakTotalNonRevocableMemoryBytes() - { - return peakTotalNonRevocableMemoryBytes; - } - @JsonProperty public long getPeakTaskUserMemory() { @@ -276,12 +263,6 @@ public double getCumulativeMemory() return cumulativeMemory; } - @JsonProperty - public double getCumulativeSystemMemory() - { - return cumulativeSystemMemory; - } - @JsonProperty public List getStageGcStatistics() { diff --git a/docs/src/main/sphinx/admin/properties-memory-management.rst b/docs/src/main/sphinx/admin/properties-memory-management.rst index 939ffc2b4488..f2fa7ca957d3 100644 --- a/docs/src/main/sphinx/admin/properties-memory-management.rst +++ b/docs/src/main/sphinx/admin/properties-memory-management.rst @@ -6,7 +6,7 @@ Memory management properties ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * **Type:** :ref:`prop-type-data-size` -* **Default value:** (JVM max memory * 0.1) +* **Default value:** (JVM max memory * 0.3) This is the max amount of user memory a query can use on a worker. User memory is allocated during execution for things that are directly @@ -15,20 +15,6 @@ by the hash tables built during execution, memory used during sorting, etc. When the user memory allocation of a query on any worker hits this limit, it is killed. -``query.max-total-memory-per-node`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* **Type:** :ref:`prop-type-data-size` -* **Default value:** (JVM max memory * 0.3) - -This is the max amount of user and system memory a query can use on a worker. -System memory is allocated during execution for things that are not directly -attributable to, or controllable by, a user query. For example, memory allocated -by the readers, writers, network buffers, etc. When the sum of the user and -system memory allocated by a query on any worker hits this limit, it is killed. -The value of ``query.max-total-memory-per-node`` must be greater than -``query.max-memory-per-node``. - ``query.max-memory`` ^^^^^^^^^^^^^^^^^^^^ @@ -48,22 +34,19 @@ it is killed. * **Type:** :ref:`prop-type-data-size` * **Default value:** (``query.max-memory`` * 2) -This is the max amount of user and system memory a query can use across the entire cluster. -System memory is allocated during execution for things that are not directly -attributable to, or controllable by, a user query. For example, memory allocated -by the readers, writers, network buffers, etc. When the sum of the user and -system memory allocated by a query across all workers hits this limit it is -killed. The value of ``query.max-total-memory`` must be greater than -``query.max-memory``. +This is the max amount of memory a query can use across the entire cluster, +including revocable memory. When the memory allocated by a query across all +workers hits this limit it is killed. The value of ``query.max-total-memory`` +must be greater than ``query.max-memory``. -``query.max-total-memory-per-task`` +``query.max-memory-per-task`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * **Type:** :ref:`prop-type-data-size` * **Default value:** none, and therefore unrestricted * **Session property:** ``query_max_total_memory_per_task`` -This is the max amount of user and system memory a task can use on a node in the +This is the max amount of the memory a task can use on a node in the cluster. Support for using this property is experimental only. ``memory.heap-headroom-per-node`` diff --git a/docs/src/main/sphinx/installation/deployment.rst b/docs/src/main/sphinx/installation/deployment.rst index f0780703cbc6..4469d02127f4 100644 --- a/docs/src/main/sphinx/installation/deployment.rst +++ b/docs/src/main/sphinx/installation/deployment.rst @@ -167,7 +167,6 @@ The following is a minimal configuration for the coordinator: http-server.http.port=8080 query.max-memory=50GB query.max-memory-per-node=1GB - query.max-total-memory-per-node=2GB discovery.uri=http://example.net:8080 And this is a minimal configuration for the workers: @@ -178,7 +177,6 @@ And this is a minimal configuration for the workers: http-server.http.port=8080 query.max-memory=50GB query.max-memory-per-node=1GB - query.max-total-memory-per-node=2GB discovery.uri=http://example.net:8080 Alternatively, if you are setting up a single machine for testing, that @@ -191,7 +189,6 @@ functions as both a coordinator and worker, use this configuration: http-server.http.port=8080 query.max-memory=5GB query.max-memory-per-node=1GB - query.max-total-memory-per-node=2GB discovery.uri=http://example.net:8080 These properties require some explanation: @@ -217,10 +214,6 @@ These properties require some explanation: * ``query.max-memory-per-node``: The maximum amount of user memory, that a query may use on any one machine. -* ``query.max-total-memory-per-node``: - The maximum amount of user and system memory, that a query may use on any one machine, - where system memory is the memory used during execution by readers, writers, and network buffers, etc. - * ``discovery.uri``: The Trino coordinator has a discovery service that is used by all the nodes to find each other. Every Trino instance registers itself with the discovery diff --git a/lib/trino-memory-context/src/main/java/io/trino/memory/context/MemoryTrackingContext.java b/lib/trino-memory-context/src/main/java/io/trino/memory/context/MemoryTrackingContext.java index 0e329783a80f..fe43d98099de 100644 --- a/lib/trino-memory-context/src/main/java/io/trino/memory/context/MemoryTrackingContext.java +++ b/lib/trino-memory-context/src/main/java/io/trino/memory/context/MemoryTrackingContext.java @@ -39,7 +39,7 @@ * and there will be system allocations initiated by the exchange clients (local allocations). All these system * allocations will be visible in the systemAggregateMemoryContext. *

- * To perform local allocations clients should use localUserMemoryContext()/localSystemMemoryContext() + * To perform local allocations clients should use localUserMemoryContext() * and get a reference to the local memory contexts. Clients can also use updateUserMemory()/tryReserveUserMemory() * to allocate memory (non-local allocations), which will be reflected to all ancestors of this context in the hierarchy. */ @@ -48,20 +48,16 @@ public final class MemoryTrackingContext { private final AggregatedMemoryContext userAggregateMemoryContext; private final AggregatedMemoryContext revocableAggregateMemoryContext; - private final AggregatedMemoryContext systemAggregateMemoryContext; private LocalMemoryContext userLocalMemoryContext; private LocalMemoryContext revocableLocalMemoryContext; - private LocalMemoryContext systemLocalMemoryContext; public MemoryTrackingContext( AggregatedMemoryContext userAggregateMemoryContext, - AggregatedMemoryContext revocableAggregateMemoryContext, - AggregatedMemoryContext systemAggregateMemoryContext) + AggregatedMemoryContext revocableAggregateMemoryContext) { this.userAggregateMemoryContext = requireNonNull(userAggregateMemoryContext, "userAggregateMemoryContext is null"); this.revocableAggregateMemoryContext = requireNonNull(revocableAggregateMemoryContext, "revocableAggregateMemoryContext is null"); - this.systemAggregateMemoryContext = requireNonNull(systemAggregateMemoryContext, "systemAggregateMemoryContext is null"); } public void close() @@ -69,10 +65,8 @@ public void close() try (Closer closer = Closer.create()) { closer.register(userAggregateMemoryContext::close); closer.register(revocableAggregateMemoryContext::close); - closer.register(systemAggregateMemoryContext::close); closer.register(userLocalMemoryContext::close); closer.register(revocableLocalMemoryContext::close); - closer.register(systemLocalMemoryContext::close); } catch (IOException e) { throw new RuntimeException("Exception closing memory tracking context", e); @@ -85,12 +79,6 @@ public LocalMemoryContext localUserMemoryContext() return userLocalMemoryContext; } - public LocalMemoryContext localSystemMemoryContext() - { - verifyNotNull(systemLocalMemoryContext, "local memory contexts are not initialized"); - return systemLocalMemoryContext; - } - public LocalMemoryContext localRevocableMemoryContext() { verifyNotNull(revocableLocalMemoryContext, "local memory contexts are not initialized"); @@ -102,11 +90,6 @@ public LocalMemoryContext newUserMemoryContext(String allocationTag) return userAggregateMemoryContext.newLocalMemoryContext(allocationTag); } - public LocalMemoryContext newSystemMemoryContext(String allocationTag) - { - return systemAggregateMemoryContext.newLocalMemoryContext(allocationTag); - } - public AggregatedMemoryContext aggregateUserMemoryContext() { return userAggregateMemoryContext; @@ -117,11 +100,6 @@ public AggregatedMemoryContext aggregateRevocableMemoryContext() return revocableAggregateMemoryContext; } - public AggregatedMemoryContext aggregateSystemMemoryContext() - { - return systemAggregateMemoryContext; - } - public AggregatedMemoryContext newAggregateUserMemoryContext() { return userAggregateMemoryContext.newAggregatedMemoryContext(); @@ -132,11 +110,6 @@ public AggregatedMemoryContext newAggregateRevocableMemoryContext() return revocableAggregateMemoryContext.newAggregatedMemoryContext(); } - public AggregatedMemoryContext newAggregateSystemMemoryContext() - { - return systemAggregateMemoryContext.newAggregatedMemoryContext(); - } - public long getUserMemory() { return userAggregateMemoryContext.getBytes(); @@ -147,28 +120,21 @@ public long getRevocableMemory() return revocableAggregateMemoryContext.getBytes(); } - public long getSystemMemory() - { - return systemAggregateMemoryContext.getBytes(); - } - public MemoryTrackingContext newMemoryTrackingContext() { return new MemoryTrackingContext( userAggregateMemoryContext.newAggregatedMemoryContext(), - revocableAggregateMemoryContext.newAggregatedMemoryContext(), - systemAggregateMemoryContext.newAggregatedMemoryContext()); + revocableAggregateMemoryContext.newAggregatedMemoryContext()); } /** * This method has to be called to initialize the local memory contexts. Otherwise, calls to methods - * localUserMemoryContext(), localSystemMemoryContext(), etc. will fail. + * localUserMemoryContext(), etc. will fail. */ public void initializeLocalMemoryContexts(String allocationTag) { this.userLocalMemoryContext = userAggregateMemoryContext.newLocalMemoryContext(allocationTag); this.revocableLocalMemoryContext = revocableAggregateMemoryContext.newLocalMemoryContext(allocationTag); - this.systemLocalMemoryContext = systemAggregateMemoryContext.newLocalMemoryContext(allocationTag); } @Override @@ -177,10 +143,8 @@ public String toString() return toStringHelper(this) .add("userAggregateMemoryContext", userAggregateMemoryContext) .add("revocableAggregateMemoryContext", revocableAggregateMemoryContext) - .add("systemAggregateMemoryContext", systemAggregateMemoryContext) .add("userLocalMemoryContext", userLocalMemoryContext) .add("revocableLocalMemoryContext", revocableLocalMemoryContext) - .add("systemLocalMemoryContext", systemLocalMemoryContext) .toString(); } } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcReader.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcReader.java index d8a592e055dd..fe0373a7186c 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcReader.java @@ -253,7 +253,7 @@ public OrcRecordReader createRecordReader( List readTypes, OrcPredicate predicate, DateTimeZone legacyFileTimeZone, - AggregatedMemoryContext systemMemoryUsage, + AggregatedMemoryContext memoryUsage, int initialBatchSize, Function exceptionTransform) throws OrcCorruptionException @@ -266,7 +266,7 @@ public OrcRecordReader createRecordReader( 0, orcDataSource.getEstimatedSize(), legacyFileTimeZone, - systemMemoryUsage, + memoryUsage, initialBatchSize, exceptionTransform, NameBasedFieldMapper::create); @@ -280,7 +280,7 @@ public OrcRecordReader createRecordReader( long offset, long length, DateTimeZone legacyFileTimeZone, - AggregatedMemoryContext systemMemoryUsage, + AggregatedMemoryContext memoryUsage, int initialBatchSize, Function exceptionTransform, FieldMapperFactory fieldMapperFactory) @@ -306,7 +306,7 @@ public OrcRecordReader createRecordReader( metadataReader, options, footer.getUserMetadata(), - systemMemoryUsage, + memoryUsage, writeValidation, initialBatchSize, exceptionTransform, diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcRecordReader.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcRecordReader.java index c2e0c82be944..77774d8b1444 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcRecordReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcRecordReader.java @@ -97,7 +97,7 @@ public class OrcRecordReader private final List stripes; private final StripeReader stripeReader; private int currentStripe = -1; - private AggregatedMemoryContext currentStripeSystemMemoryContext; + private AggregatedMemoryContext currentStripeMemoryContext; private final long fileRowCount; private final List stripeFilePositions; @@ -110,7 +110,7 @@ public class OrcRecordReader private final Map userMetadata; - private final AggregatedMemoryContext systemMemoryUsage; + private final AggregatedMemoryContext memoryUsage; private final LocalMemoryContext orcDataSourceMemoryUsage; private final OrcBlockFactory blockFactory; @@ -141,7 +141,7 @@ public OrcRecordReader( MetadataReader metadataReader, OrcReaderOptions options, Map userMetadata, - AggregatedMemoryContext systemMemoryUsage, + AggregatedMemoryContext memoryUsage, Optional writeValidation, int initialBatchSize, Function exceptionTransform, @@ -162,7 +162,7 @@ public OrcRecordReader( requireNonNull(decompressor, "decompressor is null"); requireNonNull(legacyFileTimeZone, "legacyFileTimeZone is null"); requireNonNull(userMetadata, "userMetadata is null"); - requireNonNull(systemMemoryUsage, "systemMemoryUsage is null"); + requireNonNull(memoryUsage, "memoryUsage is null"); requireNonNull(exceptionTransform, "exceptionTransform is null"); this.writeValidation = requireNonNull(writeValidation, "writeValidation is null"); @@ -170,7 +170,7 @@ public OrcRecordReader( this.rowGroupStatisticsValidation = writeValidation.map(validation -> validation.createWriteStatisticsBuilder(orcTypes, readTypes)); this.stripeStatisticsValidation = writeValidation.map(validation -> validation.createWriteStatisticsBuilder(orcTypes, readTypes)); this.fileStatisticsValidation = writeValidation.map(validation -> validation.createWriteStatisticsBuilder(orcTypes, readTypes)); - this.systemMemoryUsage = systemMemoryUsage.newAggregatedMemoryContext(); + this.memoryUsage = memoryUsage.newAggregatedMemoryContext(); this.blockFactory = new OrcBlockFactory(exceptionTransform, options.isNestedLazy()); requireNonNull(options, "options is null"); @@ -210,7 +210,7 @@ public OrcRecordReader( orcDataSource = wrapWithCacheIfTinyStripes(orcDataSource, this.stripes, options.getMaxMergeDistance(), options.getTinyStripeThreshold()); this.orcDataSource = orcDataSource; - this.orcDataSourceMemoryUsage = systemMemoryUsage.newLocalMemoryContext(OrcDataSource.class.getSimpleName()); + this.orcDataSourceMemoryUsage = memoryUsage.newLocalMemoryContext(OrcDataSource.class.getSimpleName()); this.orcDataSourceMemoryUsage.setBytes(orcDataSource.getRetainedSize()); this.splitLength = splitLength; @@ -221,13 +221,13 @@ public OrcRecordReader( this.userMetadata = ImmutableMap.copyOf(Maps.transformValues(userMetadata, Slices::copyOf)); - this.currentStripeSystemMemoryContext = this.systemMemoryUsage.newAggregatedMemoryContext(); - // The streamReadersSystemMemoryContext covers the StreamReader local buffer sizes, plus leaf node StreamReaders' + this.currentStripeMemoryContext = this.memoryUsage.newAggregatedMemoryContext(); + // The streamReadersMemoryContext covers the StreamReader local buffer sizes, plus leaf node StreamReaders' // instance sizes who use local buffers. SliceDirectStreamReader's instance size is not counted, because it // doesn't have a local buffer. All non-leaf level StreamReaders' (e.g. MapStreamReader, LongStreamReader, // ListStreamReader and StructStreamReader) instance sizes were not counted, because calling setBytes() in // their constructors is confusing. - AggregatedMemoryContext streamReadersSystemMemoryContext = this.systemMemoryUsage.newAggregatedMemoryContext(); + AggregatedMemoryContext streamReadersMemoryContext = this.memoryUsage.newAggregatedMemoryContext(); stripeReader = new StripeReader( orcDataSource, @@ -245,7 +245,7 @@ public OrcRecordReader( readColumns, readTypes, readLayouts, - streamReadersSystemMemoryContext, + streamReadersMemoryContext, blockFactory, fieldMapperFactory); @@ -496,8 +496,8 @@ private boolean advanceToNextRowGroup() private void advanceToNextStripe() throws IOException { - currentStripeSystemMemoryContext.close(); - currentStripeSystemMemoryContext = systemMemoryUsage.newAggregatedMemoryContext(); + currentStripeMemoryContext.close(); + currentStripeMemoryContext = memoryUsage.newAggregatedMemoryContext(); rowGroups = ImmutableList.of().iterator(); if (currentStripe >= 0) { @@ -521,7 +521,7 @@ private void advanceToNextStripe() StripeInformation stripeInformation = stripes.get(currentStripe); validateWriteStripe(stripeInformation.getNumberOfRows()); - Stripe stripe = stripeReader.readStripe(stripeInformation, currentStripeSystemMemoryContext); + Stripe stripe = stripeReader.readStripe(stripeInformation, currentStripeMemoryContext); if (stripe != null) { // Give readers access to dictionary streams InputStreamSources dictionaryStreamSources = stripe.getDictionaryStreamSources(); @@ -566,7 +566,7 @@ private static ColumnReader[] createColumnReaders( List columns, List readTypes, List readLayouts, - AggregatedMemoryContext systemMemoryContext, + AggregatedMemoryContext memoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) throws OrcCorruptionException @@ -576,7 +576,7 @@ private static ColumnReader[] createColumnReaders( Type readType = readTypes.get(columnIndex); OrcColumn column = columns.get(columnIndex); OrcReader.ProjectedLayout projectedLayout = readLayouts.get(columnIndex); - columnReaders[columnIndex] = createColumnReader(readType, column, projectedLayout, systemMemoryContext, blockFactory, fieldMapperFactory); + columnReaders[columnIndex] = createColumnReader(readType, column, projectedLayout, memoryContext, blockFactory, fieldMapperFactory); } return columnReaders; } @@ -602,7 +602,7 @@ long getStreamReaderRetainedSizeInBytes() @VisibleForTesting long getCurrentStripeRetainedSizeInBytes() { - return currentStripeSystemMemoryContext.getBytes(); + return currentStripeMemoryContext.getBytes(); } /** @@ -615,13 +615,13 @@ long getRetainedSizeInBytes() } /** - * @return The system memory reserved by this OrcRecordReader. It does not include non-leaf level StreamReaders' + * @return The memory reserved by this OrcRecordReader. It does not include non-leaf level StreamReaders' * instance sizes. */ @VisibleForTesting - long getSystemMemoryUsage() + long getMemoryUsage() { - return systemMemoryUsage.getBytes(); + return memoryUsage.getBytes(); } private static class StripeInfo diff --git a/lib/trino-orc/src/main/java/io/trino/orc/StripeReader.java b/lib/trino-orc/src/main/java/io/trino/orc/StripeReader.java index c59bf99ee395..068d554eaf5c 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/StripeReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/StripeReader.java @@ -113,11 +113,11 @@ public StripeReader( this.writeValidation = requireNonNull(writeValidation, "writeValidation is null"); } - public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) + public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext memoryUsage) throws IOException { // read the stripe footer - StripeFooter stripeFooter = readStripeFooter(stripe, systemMemoryUsage); + StripeFooter stripeFooter = readStripeFooter(stripe, memoryUsage); ColumnMetadata columnEncodings = stripeFooter.getColumnEncodings(); if (writeValidation.isPresent()) { writeValidation.get().validateTimeZone(orcDataSource.getId(), stripeFooter.getTimeZone()); @@ -140,7 +140,7 @@ public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext syste diskRanges = Maps.filterKeys(diskRanges, Predicates.in(streams.keySet())); // read the file regions - Map streamsData = readDiskRanges(stripe.getOffset(), diskRanges, systemMemoryUsage); + Map streamsData = readDiskRanges(stripe.getOffset(), diskRanges, memoryUsage); // read the bloom filter for each column Map> bloomFilterIndexes = readBloomFilterIndexes(streams, streamsData); @@ -157,7 +157,7 @@ public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext syste // if all row groups are skipped, return null if (selectedRowGroups.isEmpty()) { // set accounted memory usage to zero - systemMemoryUsage.close(); + memoryUsage.close(); return null; } @@ -196,7 +196,7 @@ public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext syste ImmutableMap diskRanges = diskRangesBuilder.buildOrThrow(); // read the file regions - Map streamsData = readDiskRanges(stripe.getOffset(), diskRanges, systemMemoryUsage); + Map streamsData = readDiskRanges(stripe.getOffset(), diskRanges, memoryUsage); long minAverageRowBytes = 0; for (Entry entry : streams.entrySet()) { @@ -258,7 +258,7 @@ private static boolean isSupportedStreamType(Stream stream, OrcTypeKind orcTypeK return true; } - private Map readDiskRanges(long stripeOffset, Map diskRanges, AggregatedMemoryContext systemMemoryUsage) + private Map readDiskRanges(long stripeOffset, Map diskRanges, AggregatedMemoryContext memoryUsage) throws IOException { // @@ -279,7 +279,7 @@ private Map readDiskRanges(long stripeOffset, Map dataBuilder = ImmutableMap.builder(); for (Entry entry : streamsData.entrySet()) { - dataBuilder.put(entry.getKey(), OrcChunkLoader.create(entry.getValue(), decompressor, systemMemoryUsage)); + dataBuilder.put(entry.getKey(), OrcChunkLoader.create(entry.getValue(), decompressor, memoryUsage)); } return dataBuilder.buildOrThrow(); } @@ -383,7 +383,7 @@ private static RowGroup createRowGroup(int groupId, int rowOffset, int rowCount, return new RowGroup(groupId, rowOffset, rowCount, minAverageRowBytes, rowGroupStreams); } - private StripeFooter readStripeFooter(StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) + private StripeFooter readStripeFooter(StripeInformation stripe, AggregatedMemoryContext memoryUsage) throws IOException { long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); @@ -391,7 +391,7 @@ private StripeFooter readStripeFooter(StripeInformation stripe, AggregatedMemory // read the footer Slice tailBuffer = orcDataSource.readFully(offset, tailLength); - try (InputStream inputStream = new OrcInputStream(OrcChunkLoader.create(orcDataSource.getId(), tailBuffer, decompressor, systemMemoryUsage))) { + try (InputStream inputStream = new OrcInputStream(OrcChunkLoader.create(orcDataSource.getId(), tailBuffer, decompressor, memoryUsage))) { return metadataReader.readStripeFooter(types, inputStream, legacyFileTimeZone); } } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/BooleanColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/BooleanColumnReader.java index d92e05ab59e6..64d0d9da190f 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/BooleanColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/BooleanColumnReader.java @@ -68,16 +68,16 @@ public class BooleanColumnReader private byte[] nonNullValueTemp = new byte[0]; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; - public BooleanColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMemoryContext) + public BooleanColumnReader(Type type, OrcColumn column, LocalMemoryContext memoryContext) throws OrcCorruptionException { requireNonNull(type, "type is null"); verifyStreamType(column, type, BooleanType.class::isInstance); this.column = requireNonNull(column, "column is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } @Override @@ -155,7 +155,7 @@ private Block readNullBlock(boolean[] isNull, int nonNullCount) int minNonNullValueSize = minNonNullValueSize(nonNullCount); if (nonNullValueTemp.length < minNonNullValueSize) { nonNullValueTemp = new byte[minNonNullValueSize]; - systemMemoryContext.setBytes(sizeOf(nonNullValueTemp)); + memoryContext.setBytes(sizeOf(nonNullValueTemp)); } dataStream.getSetBits(nonNullValueTemp, nonNullCount); @@ -214,7 +214,7 @@ public String toString() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/ByteColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/ByteColumnReader.java index 68db4322543e..b778ebf7a62b 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/ByteColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/ByteColumnReader.java @@ -68,16 +68,16 @@ public class ByteColumnReader private byte[] nonNullValueTemp = new byte[0]; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; - public ByteColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMemoryContext) + public ByteColumnReader(Type type, OrcColumn column, LocalMemoryContext memoryContext) throws OrcCorruptionException { requireNonNull(type, "type is null"); verifyStreamType(column, type, TinyintType.class::isInstance); this.column = requireNonNull(column, "column is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } @Override @@ -155,7 +155,7 @@ private Block readNullBlock(boolean[] isNull, int nonNullCount) int minNonNullValueSize = minNonNullValueSize(nonNullCount); if (nonNullValueTemp.length < minNonNullValueSize) { nonNullValueTemp = new byte[minNonNullValueSize]; - systemMemoryContext.setBytes(sizeOf(nonNullValueTemp)); + memoryContext.setBytes(sizeOf(nonNullValueTemp)); } dataStream.next(nonNullValueTemp, nonNullCount); @@ -215,7 +215,7 @@ public String toString() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/ColumnReaders.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/ColumnReaders.java index ba4c842d2cf2..a140a8117afe 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/ColumnReaders.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/ColumnReaders.java @@ -34,7 +34,7 @@ public static ColumnReader createColumnReader( Type type, OrcColumn column, OrcReader.ProjectedLayout projectedLayout, - AggregatedMemoryContext systemMemoryContext, + AggregatedMemoryContext memoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) throws OrcCorruptionException @@ -44,41 +44,41 @@ public static ColumnReader createColumnReader( !"TIME".equals(column.getAttributes().get("iceberg.long-type"))) { throw invalidStreamType(column, type); } - return new TimeColumnReader(type, column, systemMemoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + return new TimeColumnReader(type, column, memoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); } switch (column.getColumnType()) { case BOOLEAN: - return new BooleanColumnReader(type, column, systemMemoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + return new BooleanColumnReader(type, column, memoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); case BYTE: - return new ByteColumnReader(type, column, systemMemoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + return new ByteColumnReader(type, column, memoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); case SHORT: case INT: case LONG: case DATE: - return new LongColumnReader(type, column, systemMemoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + return new LongColumnReader(type, column, memoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); case FLOAT: - return new FloatColumnReader(type, column, systemMemoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + return new FloatColumnReader(type, column, memoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); case DOUBLE: - return new DoubleColumnReader(type, column, systemMemoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + return new DoubleColumnReader(type, column, memoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); case BINARY: case STRING: case VARCHAR: case CHAR: - return new SliceColumnReader(type, column, systemMemoryContext); + return new SliceColumnReader(type, column, memoryContext); case TIMESTAMP: case TIMESTAMP_INSTANT: - return new TimestampColumnReader(type, column, systemMemoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + return new TimestampColumnReader(type, column, memoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); case LIST: - return new ListColumnReader(type, column, systemMemoryContext, blockFactory, fieldMapperFactory); + return new ListColumnReader(type, column, memoryContext, blockFactory, fieldMapperFactory); case STRUCT: - return new StructColumnReader(type, column, projectedLayout, systemMemoryContext, blockFactory, fieldMapperFactory); + return new StructColumnReader(type, column, projectedLayout, memoryContext, blockFactory, fieldMapperFactory); case MAP: - return new MapColumnReader(type, column, systemMemoryContext, blockFactory, fieldMapperFactory); + return new MapColumnReader(type, column, memoryContext, blockFactory, fieldMapperFactory); case DECIMAL: - return new DecimalColumnReader(type, column, systemMemoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + return new DecimalColumnReader(type, column, memoryContext.newLocalMemoryContext(ColumnReaders.class.getSimpleName())); case UNION: - return new UnionColumnReader(type, column, systemMemoryContext, blockFactory, fieldMapperFactory); + return new UnionColumnReader(type, column, memoryContext, blockFactory, fieldMapperFactory); } throw new IllegalArgumentException("Unsupported type: " + column.getColumnType()); } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/DecimalColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/DecimalColumnReader.java index 21f5bea8b432..98f6c99789d3 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/DecimalColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/DecimalColumnReader.java @@ -80,9 +80,9 @@ public class DecimalColumnReader private long[] nonNullValueTemp = new long[0]; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; - public DecimalColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMemoryContext) + public DecimalColumnReader(Type type, OrcColumn column, LocalMemoryContext memoryContext) throws OrcCorruptionException { requireNonNull(type, "type is null"); @@ -90,7 +90,7 @@ public DecimalColumnReader(Type type, OrcColumn column, LocalMemoryContext syste this.type = (DecimalType) type; this.column = requireNonNull(column, "column is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } @Override @@ -225,7 +225,7 @@ private Block readShortNullBlock(boolean[] isNull, int nonNullCount) int minNonNullValueSize = minNonNullValueSize(nonNullCount); if (nonNullValueTemp.length < minNonNullValueSize) { nonNullValueTemp = new long[minNonNullValueSize]; - systemMemoryContext.setBytes(sizeOf(nonNullValueTemp)); + memoryContext.setBytes(sizeOf(nonNullValueTemp)); } decimalStream.nextShortDecimal(nonNullValueTemp, nonNullCount); @@ -251,7 +251,7 @@ private Block readLongNullBlock(boolean[] isNull, int nonNullCount) int minTempSize = minNonNullValueSize(nonNullCount) * 2; if (nonNullValueTemp.length < minTempSize) { nonNullValueTemp = new long[minTempSize]; - systemMemoryContext.setBytes(sizeOf(nonNullValueTemp)); + memoryContext.setBytes(sizeOf(nonNullValueTemp)); } decimalStream.nextLongDecimal(nonNullValueTemp, nonNullCount); @@ -343,7 +343,7 @@ public String toString() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/DoubleColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/DoubleColumnReader.java index 28c1accf9238..66c9ce5262c6 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/DoubleColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/DoubleColumnReader.java @@ -69,16 +69,16 @@ public class DoubleColumnReader private long[] nonNullValueTemp = new long[0]; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; - public DoubleColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMemoryContext) + public DoubleColumnReader(Type type, OrcColumn column, LocalMemoryContext memoryContext) throws OrcCorruptionException { requireNonNull(type, "type is null"); verifyStreamType(column, type, DoubleType.class::isInstance); this.column = requireNonNull(column, "column is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } @Override @@ -157,7 +157,7 @@ private Block readNullBlock(boolean[] isNull, int nonNullCount) int minNonNullValueSize = minNonNullValueSize(nonNullCount); if (nonNullValueTemp.length < minNonNullValueSize) { nonNullValueTemp = new long[minNonNullValueSize]; - systemMemoryContext.setBytes(sizeOf(nonNullValueTemp)); + memoryContext.setBytes(sizeOf(nonNullValueTemp)); } dataStream.next(nonNullValueTemp, nonNullCount); @@ -217,7 +217,7 @@ public String toString() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/FloatColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/FloatColumnReader.java index fa60b94512ba..c7ff0b078132 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/FloatColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/FloatColumnReader.java @@ -68,16 +68,16 @@ public class FloatColumnReader private int[] nonNullValueTemp = new int[0]; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; - public FloatColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMemoryContext) + public FloatColumnReader(Type type, OrcColumn column, LocalMemoryContext memoryContext) throws OrcCorruptionException { requireNonNull(type, "type is null"); verifyStreamType(column, type, RealType.class::isInstance); this.column = requireNonNull(column, "column is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } @Override @@ -156,7 +156,7 @@ private Block readNullBlock(boolean[] isNull, int nonNullCount) int minNonNullValueSize = minNonNullValueSize(nonNullCount); if (nonNullValueTemp.length < minNonNullValueSize) { nonNullValueTemp = new int[minNonNullValueSize]; - systemMemoryContext.setBytes(sizeOf(nonNullValueTemp)); + memoryContext.setBytes(sizeOf(nonNullValueTemp)); } dataStream.next(nonNullValueTemp, nonNullCount); @@ -216,7 +216,7 @@ public String toString() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/ListColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/ListColumnReader.java index a186c6bd2e3d..7418a62e46f0 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/ListColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/ListColumnReader.java @@ -74,7 +74,7 @@ public class ListColumnReader private boolean rowGroupOpen; - public ListColumnReader(Type type, OrcColumn column, AggregatedMemoryContext systemMemoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) + public ListColumnReader(Type type, OrcColumn column, AggregatedMemoryContext memoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) throws OrcCorruptionException { requireNonNull(type, "type is null"); @@ -87,7 +87,7 @@ public ListColumnReader(Type type, OrcColumn column, AggregatedMemoryContext sys elementType, column.getNestedColumns().get(0), fullyProjectedLayout(), - systemMemoryContext, + memoryContext, blockFactory, fieldMapperFactory); } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/LongColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/LongColumnReader.java index 119af1ece69e..484a8833dfab 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/LongColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/LongColumnReader.java @@ -80,9 +80,9 @@ public class LongColumnReader private int[] intNonNullValueTemp = new int[0]; private long[] longNonNullValueTemp = new long[0]; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; - public LongColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMemoryContext) + public LongColumnReader(Type type, OrcColumn column, LocalMemoryContext memoryContext) throws OrcCorruptionException { requireNonNull(type, "type is null"); @@ -90,7 +90,7 @@ public LongColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMe this.type = type; this.column = requireNonNull(column, "column is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } @Override @@ -205,7 +205,7 @@ private Block longReadNullBlock(boolean[] isNull, int nonNullCount) int minNonNullValueSize = minNonNullValueSize(nonNullCount); if (longNonNullValueTemp.length < minNonNullValueSize) { longNonNullValueTemp = new long[minNonNullValueSize]; - systemMemoryContext.setBytes(sizeOf(longNonNullValueTemp)); + memoryContext.setBytes(sizeOf(longNonNullValueTemp)); } dataStream.next(longNonNullValueTemp, nonNullCount); @@ -222,7 +222,7 @@ private Block intReadNullBlock(boolean[] isNull, int nonNullCount) int minNonNullValueSize = minNonNullValueSize(nonNullCount); if (intNonNullValueTemp.length < minNonNullValueSize) { intNonNullValueTemp = new int[minNonNullValueSize]; - systemMemoryContext.setBytes(sizeOf(intNonNullValueTemp)); + memoryContext.setBytes(sizeOf(intNonNullValueTemp)); } dataStream.next(intNonNullValueTemp, nonNullCount); @@ -239,7 +239,7 @@ private Block shortReadNullBlock(boolean[] isNull, int nonNullCount) int minNonNullValueSize = minNonNullValueSize(nonNullCount); if (shortNonNullValueTemp.length < minNonNullValueSize) { shortNonNullValueTemp = new short[minNonNullValueSize]; - systemMemoryContext.setBytes(sizeOf(shortNonNullValueTemp)); + memoryContext.setBytes(sizeOf(shortNonNullValueTemp)); } dataStream.next(shortNonNullValueTemp, nonNullCount); @@ -299,7 +299,7 @@ public String toString() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/MapColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/MapColumnReader.java index 5f8843d5628a..f0b9918ca480 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/MapColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/MapColumnReader.java @@ -78,7 +78,7 @@ public class MapColumnReader private boolean rowGroupOpen; - public MapColumnReader(Type type, OrcColumn column, AggregatedMemoryContext systemMemoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) + public MapColumnReader(Type type, OrcColumn column, AggregatedMemoryContext memoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) throws OrcCorruptionException { requireNonNull(type, "type is null"); @@ -91,14 +91,14 @@ public MapColumnReader(Type type, OrcColumn column, AggregatedMemoryContext syst this.type.getKeyType(), column.getNestedColumns().get(0), fullyProjectedLayout(), - systemMemoryContext, + memoryContext, blockFactory, fieldMapperFactory); this.valueColumnReader = createColumnReader( this.type.getValueType(), column.getNestedColumns().get(1), fullyProjectedLayout(), - systemMemoryContext, + memoryContext, blockFactory, fieldMapperFactory); } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/SliceColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/SliceColumnReader.java index 84ddccb193a0..a1edf4d997b9 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/SliceColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/SliceColumnReader.java @@ -53,7 +53,7 @@ public class SliceColumnReader private final SliceDictionaryColumnReader dictionaryReader; private ColumnReader currentReader; - public SliceColumnReader(Type type, OrcColumn column, AggregatedMemoryContext systemMemoryContext) + public SliceColumnReader(Type type, OrcColumn column, AggregatedMemoryContext memoryContext) throws OrcCorruptionException { requireNonNull(type, "type is null"); @@ -64,7 +64,7 @@ public SliceColumnReader(Type type, OrcColumn column, AggregatedMemoryContext sy int maxCodePointCount = getMaxCodePointCount(type); boolean charType = type instanceof CharType; directReader = new SliceDirectColumnReader(column, maxCodePointCount, charType); - dictionaryReader = new SliceDictionaryColumnReader(column, systemMemoryContext.newLocalMemoryContext(SliceColumnReader.class.getSimpleName()), maxCodePointCount, charType); + dictionaryReader = new SliceDictionaryColumnReader(column, memoryContext.newLocalMemoryContext(SliceColumnReader.class.getSimpleName()), maxCodePointCount, charType); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/SliceDictionaryColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/SliceDictionaryColumnReader.java index 432ee4823dfc..d7d8caceee90 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/SliceDictionaryColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/SliceDictionaryColumnReader.java @@ -94,14 +94,14 @@ public class SliceDictionaryColumnReader private int[] nonNullValueTemp = new int[0]; private int[] nonNullPositionList = new int[0]; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; - public SliceDictionaryColumnReader(OrcColumn column, LocalMemoryContext systemMemoryContext, int maxCodePointCount, boolean isCharType) + public SliceDictionaryColumnReader(OrcColumn column, LocalMemoryContext memoryContext, int maxCodePointCount, boolean isCharType) { this.maxCodePointCount = maxCodePointCount; this.isCharType = isCharType; this.column = requireNonNull(column, "column is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } @Override @@ -185,7 +185,7 @@ private Block readNullBlock(boolean[] isNull, int nonNullCount) if (nonNullValueTemp.length < minNonNullValueSize) { nonNullValueTemp = new int[minNonNullValueSize]; nonNullPositionList = new int[minNonNullValueSize]; - systemMemoryContext.setBytes(getRetainedSizeInBytes()); + memoryContext.setBytes(getRetainedSizeInBytes()); } dataStream.next(nonNullValueTemp, nonNullCount); @@ -219,7 +219,7 @@ private void setDictionaryBlockData(byte[] dictionaryData, int[] dictionaryOffse dictionaryOffsets[positionCount] = dictionaryOffsets[positionCount - 1]; dictionaryBlock = new VariableWidthBlock(positionCount, wrappedBuffer(dictionaryData), dictionaryOffsets, Optional.of(isNullVector)); currentDictionaryData = dictionaryData; - systemMemoryContext.setBytes(getRetainedSizeInBytes()); + memoryContext.setBytes(getRetainedSizeInBytes()); } } @@ -358,7 +358,7 @@ public String toString() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/StructColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/StructColumnReader.java index 106dcb00eebe..56b29ca9c547 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/StructColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/StructColumnReader.java @@ -80,7 +80,7 @@ public class StructColumnReader Type type, OrcColumn column, OrcReader.ProjectedLayout readLayout, - AggregatedMemoryContext systemMemoryContext, + AggregatedMemoryContext memoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) throws OrcCorruptionException @@ -112,7 +112,7 @@ public class StructColumnReader field.getType(), fieldStream, fieldLayout, - systemMemoryContext, + memoryContext, blockFactory, fieldMapperFactory)); } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/TimeColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/TimeColumnReader.java index b2b72bb91408..17e48b9e70b5 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/TimeColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/TimeColumnReader.java @@ -23,10 +23,10 @@ public class TimeColumnReader extends LongColumnReader { - public TimeColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMemoryContext) + public TimeColumnReader(Type type, OrcColumn column, LocalMemoryContext memoryContext) throws OrcCorruptionException { - super(type, column, systemMemoryContext); + super(type, column, memoryContext); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/TimestampColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/TimestampColumnReader.java index 407f4842f3f3..fbba025ff3e8 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/TimestampColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/TimestampColumnReader.java @@ -125,15 +125,15 @@ private enum TimestampKind private boolean rowGroupOpen; - private final LocalMemoryContext systemMemoryContext; + private final LocalMemoryContext memoryContext; - public TimestampColumnReader(Type type, OrcColumn column, LocalMemoryContext systemMemoryContext) + public TimestampColumnReader(Type type, OrcColumn column, LocalMemoryContext memoryContext) throws OrcCorruptionException { this.type = requireNonNull(type, "type is null"); this.column = requireNonNull(column, "column is null"); this.timestampKind = getTimestampKind(type, column); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } private static TimestampKind getTimestampKind(Type type, OrcColumn column) @@ -332,7 +332,7 @@ public String toString() @Override public void close() { - systemMemoryContext.close(); + memoryContext.close(); } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/reader/UnionColumnReader.java b/lib/trino-orc/src/main/java/io/trino/orc/reader/UnionColumnReader.java index 3f08204d8f19..9a4adce2afe5 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/reader/UnionColumnReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/reader/UnionColumnReader.java @@ -82,7 +82,7 @@ public class UnionColumnReader private boolean rowGroupOpen; - UnionColumnReader(Type type, OrcColumn column, AggregatedMemoryContext systemMemoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) + UnionColumnReader(Type type, OrcColumn column, AggregatedMemoryContext memoryContext, OrcBlockFactory blockFactory, FieldMapperFactory fieldMapperFactory) throws OrcCorruptionException { requireNonNull(type, "type is null"); @@ -99,7 +99,7 @@ public class UnionColumnReader type.getTypeParameters().get(i + 1), fields.get(i), fullyProjectedLayout(), - systemMemoryContext, + memoryContext, blockFactory, fieldMapperFactory)); } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/stream/OrcChunkLoader.java b/lib/trino-orc/src/main/java/io/trino/orc/stream/OrcChunkLoader.java index ed31bdfd821a..3fada015217e 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/stream/OrcChunkLoader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/stream/OrcChunkLoader.java @@ -29,9 +29,9 @@ static OrcChunkLoader create( OrcDataSourceId orcDataSourceId, Slice chunk, Optional decompressor, - AggregatedMemoryContext systemMemoryContext) + AggregatedMemoryContext memoryContext) { - return create(new MemoryOrcDataReader(orcDataSourceId, chunk, chunk.length()), decompressor, systemMemoryContext); + return create(new MemoryOrcDataReader(orcDataSourceId, chunk, chunk.length()), decompressor, memoryContext); } static OrcChunkLoader create( diff --git a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcReaderMemoryUsage.java b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcReaderMemoryUsage.java index 01aebcbbcaf4..10058a0144e1 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcReaderMemoryUsage.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcReaderMemoryUsage.java @@ -59,7 +59,7 @@ public void testVarcharTypeWithoutNulls() long stripeReaderRetainedSize = reader.getCurrentStripeRetainedSizeInBytes(); long streamReaderRetainedSize = reader.getStreamReaderRetainedSizeInBytes(); long readerRetainedSize = reader.getRetainedSizeInBytes(); - long readerSystemMemoryUsage = reader.getSystemMemoryUsage(); + long readerMemoryUsage = reader.getMemoryUsage(); while (true) { Page page = reader.nextPage(); @@ -78,9 +78,9 @@ public void testVarcharTypeWithoutNulls() assertGreaterThan(reader.getCurrentStripeRetainedSizeInBytes(), stripeReaderRetainedSize); // There may be some extra local buffers needed for dictionary data. assertGreaterThanOrEqual(reader.getStreamReaderRetainedSizeInBytes(), streamReaderRetainedSize); - // The total retained size and system memory usage should be greater than 0 byte because of the instance sizes. + // The total retained size and memory usage should be greater than 0 byte because of the instance sizes. assertGreaterThan(reader.getRetainedSizeInBytes() - readerRetainedSize, 0L); - assertGreaterThan(reader.getSystemMemoryUsage() - readerSystemMemoryUsage, 0L); + assertGreaterThan(reader.getMemoryUsage() - readerMemoryUsage, 0L); } } finally { @@ -104,7 +104,7 @@ public void testBigIntTypeWithNulls() long stripeReaderRetainedSize = reader.getCurrentStripeRetainedSizeInBytes(); long streamReaderRetainedSize = reader.getStreamReaderRetainedSizeInBytes(); long readerRetainedSize = reader.getRetainedSizeInBytes(); - long readerSystemMemoryUsage = reader.getSystemMemoryUsage(); + long readerMemoryUsage = reader.getMemoryUsage(); while (true) { Page page = reader.nextPage(); @@ -123,9 +123,9 @@ public void testBigIntTypeWithNulls() assertGreaterThan(reader.getCurrentStripeRetainedSizeInBytes(), stripeReaderRetainedSize); // There are no local buffers needed. assertEquals(reader.getStreamReaderRetainedSizeInBytes() - streamReaderRetainedSize, 0L); - // The total retained size and system memory usage should be strictly larger than 0L because of the instance sizes. + // The total retained size and memory usage should be strictly larger than 0L because of the instance sizes. assertGreaterThan(reader.getRetainedSizeInBytes() - readerRetainedSize, 0L); - assertGreaterThan(reader.getSystemMemoryUsage() - readerSystemMemoryUsage, 0L); + assertGreaterThan(reader.getMemoryUsage() - readerMemoryUsage, 0L); } } finally { @@ -151,7 +151,7 @@ public void testMapTypeWithNulls() long stripeReaderRetainedSize = reader.getCurrentStripeRetainedSizeInBytes(); long streamReaderRetainedSize = reader.getStreamReaderRetainedSizeInBytes(); long readerRetainedSize = reader.getRetainedSizeInBytes(); - long readerSystemMemoryUsage = reader.getSystemMemoryUsage(); + long readerMemoryUsage = reader.getMemoryUsage(); while (true) { Page page = reader.nextPage(); @@ -170,9 +170,9 @@ public void testMapTypeWithNulls() assertGreaterThan(reader.getCurrentStripeRetainedSizeInBytes(), stripeReaderRetainedSize); // There are no local buffers needed. assertEquals(reader.getStreamReaderRetainedSizeInBytes() - streamReaderRetainedSize, 0L); - // The total retained size and system memory usage should be strictly larger than 0L because of the instance sizes. + // The total retained size and memory usage should be strictly larger than 0L because of the instance sizes. assertGreaterThan(reader.getRetainedSizeInBytes() - readerRetainedSize, 0L); - assertGreaterThan(reader.getSystemMemoryUsage() - readerSystemMemoryUsage, 0L); + assertGreaterThan(reader.getMemoryUsage() - readerMemoryUsage, 0L); } } finally { @@ -278,7 +278,7 @@ private static void assertInitialRetainedSizes(OrcRecordReader reader, int rows) assertGreaterThan(reader.getStreamReaderRetainedSizeInBytes(), 0L); // there will be object overheads assertGreaterThan(reader.getRetainedSizeInBytes(), 0L); - assertEquals(reader.getSystemMemoryUsage(), 0); + assertEquals(reader.getMemoryUsage(), 0); } private static void assertClosedRetainedSizes(OrcRecordReader reader) @@ -288,6 +288,6 @@ private static void assertClosedRetainedSizes(OrcRecordReader reader) assertGreaterThan(reader.getStreamReaderRetainedSizeInBytes(), 0L); // after close() we still account for the StreamReader instance sizes. assertGreaterThan(reader.getRetainedSizeInBytes(), 0L); - assertEquals(reader.getSystemMemoryUsage(), 0); + assertEquals(reader.getMemoryUsage(), 0); } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java index 8450ce6d323c..424f4c17902e 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java @@ -91,7 +91,7 @@ public class ParquetReader private final List columns; private final ParquetDataSource dataSource; private final DateTimeZone timeZone; - private final AggregatedMemoryContext systemMemoryContext; + private final AggregatedMemoryContext memoryContext; private final Optional filter; private int currentRowGroup = -1; @@ -127,11 +127,11 @@ public ParquetReader( Optional> firstRowsOfBlocks, ParquetDataSource dataSource, DateTimeZone timeZone, - AggregatedMemoryContext systemMemoryContext, + AggregatedMemoryContext memoryContext, ParquetReaderOptions options) throws IOException { - this(fileCreatedBy, messageColumnIO, blocks, firstRowsOfBlocks, dataSource, timeZone, systemMemoryContext, options, null, null); + this(fileCreatedBy, messageColumnIO, blocks, firstRowsOfBlocks, dataSource, timeZone, memoryContext, options, null, null); } public ParquetReader( @@ -141,7 +141,7 @@ public ParquetReader( Optional> firstRowsOfBlocks, ParquetDataSource dataSource, DateTimeZone timeZone, - AggregatedMemoryContext systemMemoryContext, + AggregatedMemoryContext memoryContext, ParquetReaderOptions options, Predicate parquetPredicate, List> columnIndexStore) @@ -153,8 +153,8 @@ public ParquetReader( this.firstRowsOfBlocks = requireNonNull(firstRowsOfBlocks, "firstRowsOfBlocks is null"); this.dataSource = requireNonNull(dataSource, "dataSource is null"); this.timeZone = requireNonNull(timeZone, "timeZone is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); - this.currentRowGroupMemoryContext = systemMemoryContext.newAggregatedMemoryContext(); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); + this.currentRowGroupMemoryContext = memoryContext.newAggregatedMemoryContext(); this.options = requireNonNull(options, "options is null"); this.columnReaders = new PrimitiveColumnReader[columns.size()]; this.maxBytesPerCell = new long[columns.size()]; @@ -240,7 +240,7 @@ public int nextBatch() private boolean advanceToNextRowGroup() { currentRowGroupMemoryContext.close(); - currentRowGroupMemoryContext = systemMemoryContext.newAggregatedMemoryContext(); + currentRowGroupMemoryContext = memoryContext.newAggregatedMemoryContext(); freeCurrentRowGroupBuffers(); currentRowGroup++; if (currentRowGroup == blocks.size()) { @@ -451,9 +451,9 @@ public ParquetDataSource getDataSource() return dataSource; } - public AggregatedMemoryContext getSystemMemoryContext() + public AggregatedMemoryContext getMemoryContext() { - return systemMemoryContext; + return memoryContext; } private static List listWithNulls(int size) diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorPageSink.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorPageSink.java index 06d5ed8b7ab1..0c0331e0b01b 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorPageSink.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorPageSink.java @@ -47,10 +47,10 @@ public long getCompletedBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { - return delegate.getSystemMemoryUsage(); + return delegate.getMemoryUsage(); } } diff --git a/plugin/trino-atop/src/main/java/io/trino/plugin/atop/AtopPageSource.java b/plugin/trino-atop/src/main/java/io/trino/plugin/atop/AtopPageSource.java index 0afe9687329b..fd0bb77b814b 100644 --- a/plugin/trino-atop/src/main/java/io/trino/plugin/atop/AtopPageSource.java +++ b/plugin/trino-atop/src/main/java/io/trino/plugin/atop/AtopPageSource.java @@ -168,7 +168,7 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryEmptyProjectionPageSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryEmptyProjectionPageSource.java index 5d110ab38b1b..a57066084957 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryEmptyProjectionPageSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryEmptyProjectionPageSource.java @@ -60,7 +60,7 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryResultPageSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryResultPageSource.java index df894a10c228..5c7b8ae7d052 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryResultPageSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryResultPageSource.java @@ -279,7 +279,7 @@ private void writeBlock(BlockBuilder output, Type type, Object value) } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-blackhole/src/main/java/io/trino/plugin/blackhole/BlackHolePageSource.java b/plugin/trino-blackhole/src/main/java/io/trino/plugin/blackhole/BlackHolePageSource.java index 772faf723f6a..e9bd4d219afa 100644 --- a/plugin/trino-blackhole/src/main/java/io/trino/plugin/blackhole/BlackHolePageSource.java +++ b/plugin/trino-blackhole/src/main/java/io/trino/plugin/blackhole/BlackHolePageSource.java @@ -107,7 +107,7 @@ public long getReadTimeNanos() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return memoryUsageBytes; } diff --git a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/CountQueryPageSource.java b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/CountQueryPageSource.java index e956fd414226..7c84bf29a160 100644 --- a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/CountQueryPageSource.java +++ b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/CountQueryPageSource.java @@ -80,7 +80,7 @@ public long getCompletedBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/PassthroughQueryPageSource.java b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/PassthroughQueryPageSource.java index 4201d7aa3d2d..47e49d9c4003 100644 --- a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/PassthroughQueryPageSource.java +++ b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/PassthroughQueryPageSource.java @@ -78,7 +78,7 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/ScanQueryPageSource.java b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/ScanQueryPageSource.java index 94701c48ecd0..4baa34908f60 100644 --- a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/ScanQueryPageSource.java +++ b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/ScanQueryPageSource.java @@ -139,7 +139,7 @@ public boolean isFinished() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/FileWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/FileWriter.java index d888cc9390d9..89ed9c7a3aec 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/FileWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/FileWriter.java @@ -21,7 +21,7 @@ public interface FileWriter { long getWrittenBytes(); - long getSystemMemoryUsage(); + long getMemoryUsage(); void appendRows(Page dataPage); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSink.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSink.java index 425282280018..597526cb3b8f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSink.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSink.java @@ -93,7 +93,7 @@ public class HivePageSink private final List> verificationTasks = new ArrayList<>(); private long writtenBytes; - private long systemMemoryUsage; + private long memoryUsage; private long validationCpuNanos; public HivePageSink( @@ -174,9 +174,9 @@ public long getCompletedBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return systemMemoryUsage; + return memoryUsage; } @Override @@ -318,22 +318,22 @@ private void writePage(Page page) HiveWriter writer = writers.get(index); long currentWritten = writer.getWrittenBytes(); - long currentMemory = writer.getSystemMemoryUsage(); + long currentMemory = writer.getMemoryUsage(); writer.append(pageForWriter); writtenBytes += (writer.getWrittenBytes() - currentWritten); - systemMemoryUsage += (writer.getSystemMemoryUsage() - currentMemory); + memoryUsage += (writer.getMemoryUsage() - currentMemory); } } private void closeWriter(HiveWriter writer) { long currentWritten = writer.getWrittenBytes(); - long currentMemory = writer.getSystemMemoryUsage(); + long currentMemory = writer.getMemoryUsage(); writer.commit(); writtenBytes += (writer.getWrittenBytes() - currentWritten); - systemMemoryUsage += (writer.getSystemMemoryUsage() - currentMemory); + memoryUsage += (writer.getMemoryUsage() - currentMemory); closedWriters.add(writer); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java index 78ff27131f15..079f10631193 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java @@ -271,9 +271,9 @@ public String toString() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return delegate.getSystemMemoryUsage(); + return delegate.getMemoryUsage(); } @Override diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveStorageFormat.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveStorageFormat.java index 84ed64130fbf..d74856376e59 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveStorageFormat.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveStorageFormat.java @@ -106,14 +106,14 @@ public enum HiveStorageFormat private final String serde; private final String inputFormat; private final String outputFormat; - private final DataSize estimatedWriterSystemMemoryUsage; + private final DataSize estimatedWriterMemoryUsage; - HiveStorageFormat(String serde, String inputFormat, String outputFormat, DataSize estimatedWriterSystemMemoryUsage) + HiveStorageFormat(String serde, String inputFormat, String outputFormat, DataSize estimatedWriterMemoryUsage) { this.serde = requireNonNull(serde, "serde is null"); this.inputFormat = requireNonNull(inputFormat, "inputFormat is null"); this.outputFormat = requireNonNull(outputFormat, "outputFormat is null"); - this.estimatedWriterSystemMemoryUsage = requireNonNull(estimatedWriterSystemMemoryUsage, "estimatedWriterSystemMemoryUsage is null"); + this.estimatedWriterMemoryUsage = requireNonNull(estimatedWriterMemoryUsage, "estimatedWriterMemoryUsage is null"); } public String getSerde() @@ -131,9 +131,9 @@ public String getOutputFormat() return outputFormat; } - public DataSize getEstimatedWriterSystemMemoryUsage() + public DataSize getEstimatedWriterMemoryUsage() { - return estimatedWriterSystemMemoryUsage; + return estimatedWriterMemoryUsage; } public void validateColumns(List handles) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveUpdatablePageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveUpdatablePageSource.java index 8b8b998db5cc..995f0bef8c93 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveUpdatablePageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveUpdatablePageSource.java @@ -267,9 +267,9 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return hivePageSource.getSystemMemoryUsage(); + return hivePageSource.getMemoryUsage(); } @Override diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriter.java index 531eb4065d26..308becf8e752 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriter.java @@ -62,9 +62,9 @@ public long getWrittenBytes() return fileWriter.getWrittenBytes(); } - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return fileWriter.getSystemMemoryUsage(); + return fileWriter.getMemoryUsage(); } public long getRowCount() diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java index 61fe19118029..815e7370139b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveWriterFactory.java @@ -493,7 +493,7 @@ public HiveWriter createWriter(Page partitionColumns, int position, OptionalInt .collect(toList()), outputStorageFormat, schema, - partitionStorageFormat.getEstimatedWriterSystemMemoryUsage(), + partitionStorageFormat.getEstimatedWriterMemoryUsage(), conf, typeManager, parquetTimeZone, diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RcFileFileWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RcFileFileWriter.java index b7c500fceb7c..e0dfacdc6501 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RcFileFileWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RcFileFileWriter.java @@ -102,7 +102,7 @@ public long getWrittenBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return INSTANCE_SIZE + rcFileWriter.getRetainedSizeInBytes(); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RecordFileWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RecordFileWriter.java index 07c1bbd22d34..45e8c193b4b2 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RecordFileWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RecordFileWriter.java @@ -72,7 +72,7 @@ public class RecordFileWriter private final List structFields; private final Object row; private final FieldSetter[] setters; - private final long estimatedWriterSystemMemoryUsage; + private final long estimatedWriterMemoryUsage; private boolean committed; private long finalWrittenBytes = -1; @@ -82,7 +82,7 @@ public RecordFileWriter( List inputColumnNames, StorageFormat storageFormat, Properties schema, - DataSize estimatedWriterSystemMemoryUsage, + DataSize estimatedWriterMemoryUsage, JobConf conf, TypeManager typeManager, DateTimeZone parquetTimeZone, @@ -128,7 +128,7 @@ public RecordFileWriter( setters[i] = fieldSetterFactory.create(tableInspector, row, structFields.get(i), fileColumnTypes.get(structFields.get(i).getFieldID())); } - this.estimatedWriterSystemMemoryUsage = estimatedWriterSystemMemoryUsage.toBytes(); + this.estimatedWriterMemoryUsage = estimatedWriterMemoryUsage.toBytes(); } @Override @@ -157,9 +157,9 @@ public long getWrittenBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return INSTANCE_SIZE + estimatedWriterSystemMemoryUsage; + return INSTANCE_SIZE + estimatedWriterMemoryUsage; } @Override diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/SortingFileWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/SortingFileWriter.java index a91aa63db2d4..cb0ec82aae25 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/SortingFileWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/SortingFileWriter.java @@ -111,7 +111,7 @@ public long getWrittenBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return INSTANCE_SIZE + sortBuffer.getRetainedBytes(); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcDeleteDeltaPageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcDeleteDeltaPageSource.java index 808ece55d05f..4fb1d4a051d9 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcDeleteDeltaPageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcDeleteDeltaPageSource.java @@ -71,7 +71,7 @@ public class OrcDeleteDeltaPageSource private final OrcRecordReader recordReader; private final OrcDataSource orcDataSource; private final FileFormatDataSourceStats stats; - private final AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext(); + private final AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext(); private boolean closed; @@ -156,7 +156,7 @@ private OrcDeleteDeltaPageSource( 0, fileSize, UTC, - systemMemoryContext, + memoryContext, MAX_BATCH_SIZE, exception -> handleException(orcDataSource.getId(), exception), NameBasedFieldMapper::create); @@ -223,9 +223,9 @@ public String toString() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return systemMemoryContext.getBytes(); + return memoryContext.getBytes(); } private static String openError(Throwable t, Path path) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcDeletedRows.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcDeletedRows.java index 85981b398a15..a65933878d7e 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcDeletedRows.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcDeletedRows.java @@ -65,7 +65,7 @@ public class OrcDeletedRows private static final int BUCKET_ID_INDEX = 1; private static final int ROW_ID_INDEX = 2; - private static final long DELETED_ROWS_SYSTEM_MEMORY_INCREASE_YIELD_THREHOLD = 32 * 1204 * 1024; + private static final long DELETED_ROWS_MEMORY_INCREASE_YIELD_THREHOLD = 32 * 1204 * 1024; private final String sourceFileName; private final OrcDeleteDeltaPageSourceFactory pageSourceFactory; @@ -74,7 +74,7 @@ public class OrcDeletedRows private final HdfsEnvironment hdfsEnvironment; private final AcidInfo acidInfo; private final OptionalInt bucketNumber; - private final LocalMemoryContext systemMemoryUsage; + private final LocalMemoryContext memoryUsage; private State state = State.NOT_LOADED; @Nullable @@ -97,7 +97,7 @@ public OrcDeletedRows( HdfsEnvironment hdfsEnvironment, AcidInfo acidInfo, OptionalInt bucketNumber, - AggregatedMemoryContext systemMemoryContext) + AggregatedMemoryContext memoryContext) { this.sourceFileName = requireNonNull(sourceFileName, "sourceFileName is null"); this.pageSourceFactory = requireNonNull(pageSourceFactory, "pageSourceFactory is null"); @@ -106,7 +106,7 @@ public OrcDeletedRows( this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.acidInfo = requireNonNull(acidInfo, "acidInfo is null"); this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.systemMemoryUsage = requireNonNull(systemMemoryContext, "systemMemoryContext is null").newLocalMemoryContext(OrcDeletedRows.class.getSimpleName()); + this.memoryUsage = requireNonNull(memoryContext, "memoryContext is null").newLocalMemoryContext(OrcDeletedRows.class.getSimpleName()); } public MaskDeletedRowsFunction getMaskDeletedRowsFunction(Page sourcePage, OptionalLong startRowId) @@ -353,8 +353,8 @@ public Optional> loadOrYield() if (deletedRowsBuilderSize % 1000 == 0) { long currentMemorySize = retainedMemorySize(deletedRowsBuilderSize, currentPage); - if (currentMemorySize - initialMemorySize >= DELETED_ROWS_SYSTEM_MEMORY_INCREASE_YIELD_THREHOLD) { - systemMemoryUsage.setBytes(currentMemorySize); + if (currentMemorySize - initialMemorySize >= DELETED_ROWS_MEMORY_INCREASE_YIELD_THREHOLD) { + memoryUsage.setBytes(currentMemorySize); return Optional.empty(); } } @@ -379,7 +379,7 @@ public Optional> loadOrYield() } Set builtDeletedRows = deletedRowsBuilder.build(); - systemMemoryUsage.setBytes(retainedMemorySize(builtDeletedRows.size(), null)); + memoryUsage.setBytes(retainedMemorySize(builtDeletedRows.size(), null)); return Optional.of(builtDeletedRows); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriter.java index 53bd73f66c97..416f85e4ec92 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriter.java @@ -136,7 +136,7 @@ public long getWrittenBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return INSTANCE_SIZE + orcWriter.getRetainedBytes(); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSource.java index 2e188bd3260d..59022fe6da10 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSource.java @@ -72,7 +72,7 @@ public class OrcPageSource private boolean closed; - private final AggregatedMemoryContext systemMemoryContext; + private final AggregatedMemoryContext memoryContext; private final LocalMemoryContext localMemoryContext; private final FileFormatDataSourceStats stats; @@ -90,7 +90,7 @@ public OrcPageSource( OrcDataSource orcDataSource, Optional deletedRows, Optional originalFileRowId, - AggregatedMemoryContext systemMemoryContext, + AggregatedMemoryContext memoryContext, FileFormatDataSourceStats stats) { this.recordReader = requireNonNull(recordReader, "recordReader is null"); @@ -98,8 +98,8 @@ public OrcPageSource( this.orcDataSource = requireNonNull(orcDataSource, "orcDataSource is null"); this.deletedRows = requireNonNull(deletedRows, "deletedRows is null"); this.stats = requireNonNull(stats, "stats is null"); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); - this.localMemoryContext = systemMemoryContext.newLocalMemoryContext(OrcPageSource.class.getSimpleName()); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); + this.localMemoryContext = memoryContext.newLocalMemoryContext(OrcPageSource.class.getSimpleName()); this.originalFileRowId = requireNonNull(originalFileRowId, "originalFileRowId is null"); } @@ -240,9 +240,9 @@ public String toString() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return systemMemoryContext.getBytes(); + return memoryContext.getBytes(); } public interface ColumnAdaptation diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java index dd6a8acc48c7..8da7790fca80 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java @@ -268,7 +268,7 @@ private ConnectorPageSource createOrcPageSource( throw new TrinoException(HIVE_CANNOT_OPEN_SPLIT, splitError(e, path, start, length), e); } - AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); + AggregatedMemoryContext memoryUsage = newSimpleAggregatedMemoryContext(); try { Optional optionalOrcReader = OrcReader.createOrcReader(orcDataSource, options); if (optionalOrcReader.isEmpty()) { @@ -388,7 +388,7 @@ else if (column.getBaseHiveColumnIndex() < fileColumns.size()) { start, length, legacyFileTimeZone, - systemMemoryUsage, + memoryUsage, INITIAL_BATCH_SIZE, exception -> handleException(orcDataSource.getId(), exception), NameBasedFieldMapper::create); @@ -402,7 +402,7 @@ else if (column.getBaseHiveColumnIndex() < fileColumns.size()) { hdfsEnvironment, info, bucketNumber, - systemMemoryUsage)); + memoryUsage)); Optional originalFileRowId = acidInfo .filter(OrcPageSourceFactory::hasOriginalFiles) @@ -447,7 +447,7 @@ else if (transaction.isUpdate()) { orcDataSource, deletedRows, originalFileRowId, - systemMemoryUsage, + memoryUsage, stats); } catch (Exception e) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetFileWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetFileWriter.java index 52a10ff7f00d..b3681a0544a1 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetFileWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetFileWriter.java @@ -90,7 +90,7 @@ public long getWrittenBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return INSTANCE_SIZE + parquetWriter.getRetainedBytes(); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java index e4d921793794..758d29e457f7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java @@ -124,9 +124,9 @@ public boolean isFinished() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return parquetReader.getSystemMemoryContext().getBytes(); + return parquetReader.getMemoryContext().getBytes(); } @Override diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/rcfile/RcFilePageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/rcfile/RcFilePageSource.java index 074ca42f8b65..4ed97f6fab92 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/rcfile/RcFilePageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/rcfile/RcFilePageSource.java @@ -175,7 +175,7 @@ public String toString() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return GUESSED_MEMORY_USAGE; } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/ForwardingRecordCursor.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/ForwardingRecordCursor.java index b62bb76cc877..3f1a410cd3f5 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/ForwardingRecordCursor.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/ForwardingRecordCursor.java @@ -83,9 +83,9 @@ public boolean isNull(int field) } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return delegate().getSystemMemoryUsage(); + return delegate().getMemoryUsage(); } @Override diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOrcPageSourceMemoryTracking.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOrcPageSourceMemoryTracking.java index ae7e40536516..35d078de8340 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOrcPageSourceMemoryTracking.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOrcPageSourceMemoryTracking.java @@ -198,10 +198,10 @@ private void testPageSource(boolean useCache) if (useCache) { // file is fully cached - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 200); + assertBetweenInclusive(pageSource.getMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 200); } else { - assertEquals(pageSource.getSystemMemoryUsage(), 0); + assertEquals(pageSource.getMemoryUsage(), 0); } long memoryUsage = -1; @@ -215,25 +215,25 @@ private void testPageSource(boolean useCache) if (memoryUsage == -1) { // Memory usage before lazy-loading the block if (useCache) { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 2000); + assertBetweenInclusive(pageSource.getMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 2000); } else { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), 0L, 1000L); + assertBetweenInclusive(pageSource.getMemoryUsage(), 0L, 1000L); } createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block - memoryUsage = pageSource.getSystemMemoryUsage(); + memoryUsage = pageSource.getMemoryUsage(); // Memory usage after lazy-loading the actual block if (useCache) { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), testPreparer.getFileSize() + 270_000, testPreparer.getFileSize() + 280_000); + assertBetweenInclusive(pageSource.getMemoryUsage(), testPreparer.getFileSize() + 270_000, testPreparer.getFileSize() + 280_000); } else { assertBetweenInclusive(memoryUsage, 460_000L, 469_999L); } } else { - assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); + assertEquals(pageSource.getMemoryUsage(), memoryUsage); createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block - assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); + assertEquals(pageSource.getMemoryUsage(), memoryUsage); } totalRows += page.getPositionCount(); } @@ -248,25 +248,25 @@ private void testPageSource(boolean useCache) if (memoryUsage == -1) { // Memory usage before lazy-loading the block if (useCache) { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 2000); + assertBetweenInclusive(pageSource.getMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 2000); } else { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), 0L, 1000L); + assertBetweenInclusive(pageSource.getMemoryUsage(), 0L, 1000L); } createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block - memoryUsage = pageSource.getSystemMemoryUsage(); + memoryUsage = pageSource.getMemoryUsage(); // Memory usage after lazy-loading the actual block if (useCache) { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), testPreparer.getFileSize() + 270_000, testPreparer.getFileSize() + 280_000); + assertBetweenInclusive(pageSource.getMemoryUsage(), testPreparer.getFileSize() + 270_000, testPreparer.getFileSize() + 280_000); } else { assertBetweenInclusive(memoryUsage, 460_000L, 469_999L); } } else { - assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); + assertEquals(pageSource.getMemoryUsage(), memoryUsage); createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block - assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); + assertEquals(pageSource.getMemoryUsage(), memoryUsage); } totalRows += page.getPositionCount(); } @@ -281,25 +281,25 @@ private void testPageSource(boolean useCache) if (memoryUsage == -1) { // Memory usage before lazy-loading the block if (useCache) { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 2000); + assertBetweenInclusive(pageSource.getMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 2000); } else { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), 0L, 1000L); + assertBetweenInclusive(pageSource.getMemoryUsage(), 0L, 1000L); } createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block - memoryUsage = pageSource.getSystemMemoryUsage(); + memoryUsage = pageSource.getMemoryUsage(); // Memory usage after loading the actual block if (useCache) { - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), testPreparer.getFileSize() + 260_000, testPreparer.getFileSize() + 270_000); + assertBetweenInclusive(pageSource.getMemoryUsage(), testPreparer.getFileSize() + 260_000, testPreparer.getFileSize() + 270_000); } else { assertBetweenInclusive(memoryUsage, 360_000L, 369_999L); } } else { - assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); + assertEquals(pageSource.getMemoryUsage(), memoryUsage); createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block - assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); + assertEquals(pageSource.getMemoryUsage(), memoryUsage); } totalRows += page.getPositionCount(); } @@ -309,10 +309,10 @@ private void testPageSource(boolean useCache) assertTrue(pageSource.isFinished()); if (useCache) { // file is fully cached - assertBetweenInclusive(pageSource.getSystemMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 200); + assertBetweenInclusive(pageSource.getMemoryUsage(), testPreparer.getFileSize(), testPreparer.getFileSize() + 200); } else { - assertEquals(pageSource.getSystemMemoryUsage(), 0); + assertEquals(pageSource.getMemoryUsage(), 0); } pageSource.close(); } @@ -391,7 +391,7 @@ public void testTableScanOperator() DriverContext driverContext = testPreparer.newDriverContext(); SourceOperator operator = testPreparer.newTableScanOperator(driverContext); - assertEquals(driverContext.getSystemMemoryUsage(), 0); + assertEquals(driverContext.getMemoryUsage(), 0); long memoryUsage = -1; int totalRows = 0; @@ -401,11 +401,11 @@ public void testTableScanOperator() assertNotNull(page); page.getBlock(1); if (memoryUsage == -1) { - memoryUsage = driverContext.getSystemMemoryUsage(); + memoryUsage = driverContext.getMemoryUsage(); assertBetweenInclusive(memoryUsage, 460000L, 469999L); } else { - assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage); + assertEquals(driverContext.getMemoryUsage(), memoryUsage); } totalRows += page.getPositionCount(); } @@ -417,11 +417,11 @@ public void testTableScanOperator() assertNotNull(page); page.getBlock(1); if (memoryUsage == -1) { - memoryUsage = driverContext.getSystemMemoryUsage(); + memoryUsage = driverContext.getMemoryUsage(); assertBetweenInclusive(memoryUsage, 460000L, 469999L); } else { - assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage); + assertEquals(driverContext.getMemoryUsage(), memoryUsage); } totalRows += page.getPositionCount(); } @@ -433,11 +433,11 @@ public void testTableScanOperator() assertNotNull(page); page.getBlock(1); if (memoryUsage == -1) { - memoryUsage = driverContext.getSystemMemoryUsage(); + memoryUsage = driverContext.getMemoryUsage(); assertBetweenInclusive(memoryUsage, 360000L, 369999L); } else { - assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage); + assertEquals(driverContext.getMemoryUsage(), memoryUsage); } totalRows += page.getPositionCount(); } @@ -445,7 +445,7 @@ public void testTableScanOperator() assertFalse(operator.isFinished()); assertNull(operator.getOutput()); assertTrue(operator.isFinished()); - assertEquals(driverContext.getSystemMemoryUsage(), 0); + assertEquals(driverContext.getMemoryUsage(), 0); } @Test @@ -457,7 +457,7 @@ public void testScanFilterAndProjectOperator() DriverContext driverContext = testPreparer.newDriverContext(); SourceOperator operator = testPreparer.newScanFilterAndProjectOperator(driverContext); - assertEquals(driverContext.getSystemMemoryUsage(), 0); + assertEquals(driverContext.getMemoryUsage(), 0); int totalRows = 0; while (totalRows < NUM_ROWS) { @@ -466,7 +466,7 @@ public void testScanFilterAndProjectOperator() assertNotNull(page); // memory usage varies depending on stripe alignment - long memoryUsage = driverContext.getSystemMemoryUsage(); + long memoryUsage = driverContext.getMemoryUsage(); assertTrue(memoryUsage < 1000 || (memoryUsage > 350_000L && memoryUsage < 465_000L)); totalRows += page.getPositionCount(); @@ -475,7 +475,7 @@ public void testScanFilterAndProjectOperator() // done... in the current implementation finish is not set until output returns a null page assertNull(operator.getOutput()); assertTrue(operator.isFinished()); - assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 0L, 500L); + assertBetweenInclusive(driverContext.getMemoryUsage(), 0L, 500L); } private class TestPreparer diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/StandardFileFormats.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/StandardFileFormats.java index e06ae51f0a78..e6483e0820ed 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/StandardFileFormats.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/StandardFileFormats.java @@ -414,7 +414,7 @@ public RecordFormatWriter( columnNames, fromHiveStorageFormat(format), createSchema(format, columnNames, columnTypes), - format.getEstimatedWriterSystemMemoryUsage(), + format.getEstimatedWriterMemoryUsage(), config, TESTING_TYPE_MANAGER, UTC, diff --git a/plugin/trino-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpQueryListener.java b/plugin/trino-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpQueryListener.java index b32af2cf06fa..18a8a395c0ba 100644 --- a/plugin/trino-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpQueryListener.java +++ b/plugin/trino-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpQueryListener.java @@ -153,8 +153,6 @@ public class TestHttpQueryListener 0L, 0L, 0L, - 0L, - 0.0f, 0.0f, Collections.emptyList(), 0, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSink.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSink.java index 57af22553085..38ea45218b3d 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSink.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSink.java @@ -102,7 +102,7 @@ public class IcebergPageSink private final List writers = new ArrayList<>(); private long writtenBytes; - private long systemMemoryUsage; + private long memoryUsage; private long validationCpuNanos; public IcebergPageSink( @@ -143,9 +143,9 @@ public long getCompletedBytes() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return systemMemoryUsage; + return memoryUsage; } @Override @@ -264,12 +264,12 @@ private void writePage(Page page) IcebergFileWriter writer = writers.get(index).getWriter(); long currentWritten = writer.getWrittenBytes(); - long currentMemory = writer.getSystemMemoryUsage(); + long currentMemory = writer.getMemoryUsage(); writer.appendRows(pageForWriter); writtenBytes += (writer.getWrittenBytes() - currentWritten); - systemMemoryUsage += (writer.getSystemMemoryUsage() - currentMemory); + memoryUsage += (writer.getMemoryUsage() - currentMemory); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSource.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSource.java index 586e954adf92..837bca2ca020 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSource.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSource.java @@ -147,9 +147,9 @@ public String toString() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return delegate.getSystemMemoryUsage(); + return delegate.getMemoryUsage(); } protected void closeWithSuppression(Throwable throwable) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java index e31fec31d64b..9983e08a989b 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java @@ -373,7 +373,7 @@ private static ReaderPageSource createOrcPageSource( } } - AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); + AggregatedMemoryContext memoryUsage = newSimpleAggregatedMemoryContext(); OrcDataSourceId orcDataSourceId = orcDataSource.getId(); OrcRecordReader recordReader = reader.createRecordReader( fileReadColumns, @@ -383,7 +383,7 @@ private static ReaderPageSource createOrcPageSource( start, length, UTC, - systemMemoryUsage, + memoryUsage, INITIAL_BATCH_SIZE, exception -> handleException(orcDataSourceId, exception), new IdBasedFieldMapperFactory(readColumns)); @@ -395,7 +395,7 @@ private static ReaderPageSource createOrcPageSource( orcDataSource, Optional.empty(), Optional.empty(), - systemMemoryUsage, + memoryUsage, stats), columnProjections); } @@ -611,7 +611,7 @@ private static ReaderPageSource createParquetPageSource( FileFormatDataSourceStats fileFormatDataSourceStats, Optional nameMapping) { - AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext(); + AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext(); ParquetDataSource dataSource = null; try { @@ -663,7 +663,7 @@ private static ReaderPageSource createParquetPageSource( Optional.empty(), dataSource, UTC, - systemMemoryContext, + memoryContext, options); ImmutableList.Builder trinoTypes = ImmutableList.builder(); diff --git a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduUpdatablePageSource.java b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduUpdatablePageSource.java index e761e794f72d..d3ec50f2ac09 100644 --- a/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduUpdatablePageSource.java +++ b/plugin/trino-kudu/src/main/java/io/trino/plugin/kudu/KuduUpdatablePageSource.java @@ -104,9 +104,9 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return inner.getSystemMemoryUsage(); + return inner.getMemoryUsage(); } @Override diff --git a/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryPageSourceProvider.java b/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryPageSourceProvider.java index ee4ce4c3ed45..e22af7c5b39f 100644 --- a/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryPageSourceProvider.java +++ b/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryPageSourceProvider.java @@ -163,9 +163,9 @@ public CompletableFuture isBlocked() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return delegate.getSystemMemoryUsage(); + return delegate.getMemoryUsage(); } @Override diff --git a/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoPageSource.java b/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoPageSource.java index 4feb1f73916a..a62460c47825 100644 --- a/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoPageSource.java +++ b/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoPageSource.java @@ -127,7 +127,7 @@ public boolean isFinished() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0L; } diff --git a/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotBrokerPageSource.java b/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotBrokerPageSource.java index e422bae25a21..798fb2af43a8 100644 --- a/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotBrokerPageSource.java +++ b/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotBrokerPageSource.java @@ -149,7 +149,7 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotSegmentPageSource.java b/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotSegmentPageSource.java index bde45122c292..5213508e5573 100755 --- a/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotSegmentPageSource.java +++ b/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotSegmentPageSource.java @@ -122,7 +122,7 @@ public long getReadTimeNanos() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return estimatedMemoryUsageInBytes; } diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java index 0ab12ea9a2e3..6decc7e2eff1 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java @@ -58,7 +58,7 @@ public class RaptorPageSource private final BitSet rowsToDelete; - private final AggregatedMemoryContext systemMemoryContext; + private final AggregatedMemoryContext memoryContext; private boolean closed; @@ -67,7 +67,7 @@ public RaptorPageSource( OrcRecordReader recordReader, List columnAdaptations, OrcDataSource orcDataSource, - AggregatedMemoryContext systemMemoryContext) + AggregatedMemoryContext memoryContext) { this.shardRewriter = requireNonNull(shardRewriter, "shardRewriter is null"); this.recordReader = requireNonNull(recordReader, "recordReader is null"); @@ -76,7 +76,7 @@ public RaptorPageSource( this.rowsToDelete = new BitSet(toIntExact(recordReader.getFileRowCount())); - this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); } @Override @@ -171,9 +171,9 @@ public CompletableFuture> finish() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return systemMemoryContext.getBytes(); + return memoryContext.getBytes(); } public interface ColumnAdaptation diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java index 95fd3bbccb0d..31c09acdd44c 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java @@ -240,7 +240,7 @@ public ConnectorPageSource getPageSource( orcReaderOptions = orcReaderOptions.withMaxReadBlockSize(HUGE_MAX_READ_BLOCK_SIZE); OrcDataSource dataSource = openShard(shardUuid, orcReaderOptions); - AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); + AggregatedMemoryContext memoryUsage = newSimpleAggregatedMemoryContext(); try { OrcReader reader = OrcReader.createOrcReader(dataSource, orcReaderOptions) @@ -278,7 +278,7 @@ public ConnectorPageSource getPageSource( fileReadTypes, predicate, UTC, - systemMemoryUsage, + memoryUsage, INITIAL_BATCH_SIZE, RaptorPageSource::handleException); @@ -287,7 +287,7 @@ public ConnectorPageSource getPageSource( shardRewriter = Optional.of(createShardRewriter(transactionId.getAsLong(), bucketNumber, shardUuid)); } - return new RaptorPageSource(shardRewriter, recordReader, columnAdaptations, dataSource, systemMemoryUsage); + return new RaptorPageSource(shardRewriter, recordReader, columnAdaptations, dataSource, memoryUsage); } catch (IOException | RuntimeException e) { closeQuietly(dataSource); diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java index ae382c5b9169..3d346df637ef 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java @@ -76,9 +76,9 @@ public Page getNextPage() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return (current != null) ? current.getSystemMemoryUsage() : 0; + return (current != null) ? current.getMemoryUsage() : 0; } @Override diff --git a/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftIndexPageSource.java b/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftIndexPageSource.java index 3050d550c2f5..4b31f0a44861 100644 --- a/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftIndexPageSource.java +++ b/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftIndexPageSource.java @@ -150,7 +150,7 @@ public long getReadTimeNanos() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftPageSource.java b/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftPageSource.java index e080231232bb..93d4724460d7 100644 --- a/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftPageSource.java +++ b/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftPageSource.java @@ -113,7 +113,7 @@ public long getReadTimeNanos() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { return 0; } diff --git a/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/LazyRecordPageSource.java b/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/LazyRecordPageSource.java index fe19eec42742..cf3bb333f57f 100644 --- a/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/LazyRecordPageSource.java +++ b/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/LazyRecordPageSource.java @@ -66,9 +66,9 @@ public long getReadTimeNanos() } @Override - public long getSystemMemoryUsage() + public long getMemoryUsage() { - return cursor.getSystemMemoryUsage() + pageBuilder.getSizeInBytes(); + return cursor.getMemoryUsage() + pageBuilder.getSizeInBytes(); } @Override diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/AbstractOperatorBenchmark.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/AbstractOperatorBenchmark.java index 6dad9df645f8..3cc81eb85a08 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/AbstractOperatorBenchmark.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/AbstractOperatorBenchmark.java @@ -301,7 +301,6 @@ protected Map runOnce() TaskContext taskContext = new QueryContext( new QueryId("test"), DataSize.of(256, MEGABYTE), - DataSize.of(512, MEGABYTE), Optional.empty(), memoryPool, new TestingGcMonitor(), diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/CountAggregationBenchmark.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/CountAggregationBenchmark.java index f6f1fa87ec99..2211dc5fbbe1 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/CountAggregationBenchmark.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/CountAggregationBenchmark.java @@ -41,8 +41,7 @@ protected List createOperatorFactories() AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory( 1, new PlanNodeId("test"), - ImmutableList.of(countFunction.bind(ImmutableList.of(0))), - false); + ImmutableList.of(countFunction.bind(ImmutableList.of(0)))); return ImmutableList.of(tableScanOperator, aggregationOperator); } diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/DoubleSumAggregationBenchmark.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/DoubleSumAggregationBenchmark.java index 7d95161efcf6..98e41577e354 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/DoubleSumAggregationBenchmark.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/DoubleSumAggregationBenchmark.java @@ -42,8 +42,7 @@ protected List createOperatorFactories() AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory( 1, new PlanNodeId("test"), - ImmutableList.of(doubleSum.bind(ImmutableList.of(0))), - false); + ImmutableList.of(doubleSum.bind(ImmutableList.of(0)))); return ImmutableList.of(tableScanOperator, aggregationOperator); } diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HandTpchQuery1.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HandTpchQuery1.java index fa344cb282fd..1d5302c975c5 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HandTpchQuery1.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HandTpchQuery1.java @@ -121,8 +121,7 @@ protected List createOperatorFactories() 10_000, Optional.of(DataSize.of(16, MEGABYTE)), new JoinCompiler(localQueryRunner.getTypeOperators()), - localQueryRunner.getBlockTypeOperators(), - false); + localQueryRunner.getBlockTypeOperators()); return ImmutableList.of(tableScanOperator, tpchQuery1Operator, aggregationOperator); } diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HandTpchQuery6.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HandTpchQuery6.java index 7dd25dba33b7..e22b3d60d978 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HandTpchQuery6.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HandTpchQuery6.java @@ -78,8 +78,7 @@ protected List createOperatorFactories() 2, new PlanNodeId("test"), ImmutableList.of( - doubleSum.bind(ImmutableList.of(0))), - false); + doubleSum.bind(ImmutableList.of(0)))); return ImmutableList.of(tableScanOperator, tpchQuery6Operator, aggregationOperator); } diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashAggregationBenchmark.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashAggregationBenchmark.java index 75eeb7ff6099..7105a6e74f7c 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashAggregationBenchmark.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashAggregationBenchmark.java @@ -61,8 +61,7 @@ protected List createOperatorFactories() 100_000, Optional.of(DataSize.of(16, MEGABYTE)), new JoinCompiler(localQueryRunner.getTypeOperators()), - localQueryRunner.getBlockTypeOperators(), - false); + localQueryRunner.getBlockTypeOperators()); return ImmutableList.of(tableScanOperator, aggregationOperator); } diff --git a/testing/trino-benchmark/src/test/java/io/trino/benchmark/MemoryLocalQueryRunner.java b/testing/trino-benchmark/src/test/java/io/trino/benchmark/MemoryLocalQueryRunner.java index 6aeb566e47b1..ccba74ab1b99 100644 --- a/testing/trino-benchmark/src/test/java/io/trino/benchmark/MemoryLocalQueryRunner.java +++ b/testing/trino-benchmark/src/test/java/io/trino/benchmark/MemoryLocalQueryRunner.java @@ -73,7 +73,6 @@ public List execute(@Language("SQL") String query) QueryContext queryContext = new QueryContext( new QueryId("test"), DataSize.of(1, GIGABYTE), - DataSize.of(2, GIGABYTE), Optional.empty(), memoryPool, new TestingGcMonitor(), diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeCompatibility.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeCompatibility.java index 6ed5c424215b..8ca642f810ea 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeCompatibility.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeCompatibility.java @@ -69,11 +69,12 @@ public void extendEnvironment(Environment.Builder builder, Map e private void configureCompatibilityTestContainer(Environment.Builder builder, Config config) { - String containerConfigDir = getConfigurationDirectory(config.getCompatibilityTestDockerImage()); - DockerContainer container = new DockerContainer(config.getCompatibilityTestDockerImage(), COMPATIBILTY_TEST_CONTAINER_NAME) + String dockerImage = config.getCompatibilityTestDockerImage(); + String containerConfigDir = getConfigurationDirectory(dockerImage); + DockerContainer container = new DockerContainer(dockerImage, COMPATIBILTY_TEST_CONTAINER_NAME) .withExposedLogPaths("/var/trino/var/log", "/var/log/container-health.log") .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("conf/presto/etc/jvm.config")), containerConfigDir + "jvm.config") - .withCopyFileToContainer(forHostPath(configDir.getPath("config.properties")), containerConfigDir + "config.properties") + .withCopyFileToContainer(forHostPath(configDir.getPath(getConfigFileFor(dockerImage))), containerConfigDir + "config.properties") .withCopyFileToContainer(forHostPath(configDir.getPath("hive.properties")), containerConfigDir + "catalog/hive.properties") .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath()), "/docker/presto-product-tests") .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) @@ -83,10 +84,10 @@ private void configureCompatibilityTestContainer(Environment.Builder builder, Co portBinder.exposePort(container, SERVER_PORT); } - protected String getConfigurationDirectory(String dockerImageName) + protected String getConfigurationDirectory(String dockerImage) { try { - int version = getVersionFromDockerImageName(dockerImageName); + int version = getVersionFromDockerImageName(dockerImage); if (version <= 350) { return "/usr/lib/presto/default/etc/"; } @@ -97,10 +98,18 @@ protected String getConfigurationDirectory(String dockerImageName) return "/etc/trino/"; } catch (NumberFormatException e) { - throw new RuntimeException("Failed to parse version from docker image name " + dockerImageName); + throw new RuntimeException("Failed to parse version from docker image name " + dockerImage); } } + private String getConfigFileFor(String dockerImage) + { + if (getVersionFromDockerImageName(dockerImage) < 369) { + return "config-with-system-memory.properties"; + } + return "config.properties"; + } + private void configureTestsContainer(Environment.Builder builder, Config config) { int version = getVersionFromDockerImageName(config.getCompatibilityTestDockerImage()); diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/config.properties index 306b335bf4f6..05b292005d11 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/config.properties @@ -5,8 +5,7 @@ node.internal-address-source=FQDN coordinator=true node-scheduler.include-coordinator=true query.max-memory=1GB -query.max-memory-per-node=512MB -query.max-total-memory-per-node=1GB +query.max-memory-per-node=1GB discovery.uri=https://presto-master.docker.cluster:7778 http.authentication.krb5.config=/etc/krb5.conf diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard-multinode/multinode-worker-config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard-multinode/multinode-worker-config.properties index d237f5addeb0..3e4d76ea1bd9 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard-multinode/multinode-worker-config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard-multinode/multinode-worker-config.properties @@ -5,6 +5,5 @@ coordinator=false experimental.concurrent-startup=true http-server.http.port=8081 query.max-memory=1GB -query.max-memory-per-node=768MB -query.max-total-memory-per-node=1GB +query.max-memory-per-node=1GB discovery.uri=http://presto-master:8080 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard/config.properties index 57a852184573..a1abe821738d 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard/config.properties @@ -6,6 +6,5 @@ experimental.concurrent-startup=true node-scheduler.include-coordinator=true http-server.http.port=8080 query.max-memory=2GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=1.25GB +query.max-memory-per-node=1.25GB discovery.uri=http://presto-master:8080 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls/config-master.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls/config-master.properties index 91b174147a6b..256e604a668b 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls/config-master.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls/config-master.properties @@ -7,8 +7,7 @@ node-scheduler.include-coordinator=true discovery.uri=https://presto-master.docker.cluster:7778 query.max-memory=1GB -query.max-memory-per-node=512MB -query.max-total-memory-per-node=1GB +query.max-memory-per-node=1GB http-server.http.enabled=false http-server.https.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls/config-worker.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls/config-worker.properties index 64865a888026..bc46ddcb33f1 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls/config-worker.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls/config-worker.properties @@ -6,8 +6,7 @@ coordinator=false discovery.uri=https://presto-master.docker.cluster:7778 query.max-memory=1GB -query.max-memory-per-node=512MB -query.max-total-memory-per-node=1GB +query.max-memory-per-node=1GB http-server.http.enabled=false http-server.https.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/config-with-system-memory.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/config-with-system-memory.properties new file mode 100644 index 000000000000..951e4923b1f5 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/config-with-system-memory.properties @@ -0,0 +1,8 @@ +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8081 +query.max-memory=2GB +query.max-memory-per-node=1GB +query.max-total-memory-per-node=1.25GB +discovery-server.enabled=true +discovery.uri=http://compatibility-test-coordinator:8081 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/config.properties index 951e4923b1f5..9822f0100f35 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/config.properties @@ -2,7 +2,6 @@ coordinator=true node-scheduler.include-coordinator=true http-server.http.port=8081 query.max-memory=2GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=1.25GB +query.max-memory-per-node=1.25GB discovery-server.enabled=true discovery.uri=http://compatibility-test-coordinator:8081 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-ldap-and-file/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-ldap-and-file/config.properties index 2716238329df..9574860d25d2 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-ldap-and-file/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-ldap-and-file/config.properties @@ -12,8 +12,7 @@ node.internal-address-source=FQDN coordinator=true node-scheduler.include-coordinator=true query.max-memory=1GB -query.max-memory-per-node=512MB -query.max-total-memory-per-node=1GB +query.max-memory-per-node=1GB discovery.uri=https://presto-master:8443 # LDAP specific properties diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-ldap/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-ldap/config.properties index ea0dcefbe424..c4b994bae711 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-ldap/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-ldap/config.properties @@ -12,8 +12,7 @@ node.internal-address-source=FQDN coordinator=true node-scheduler.include-coordinator=true query.max-memory=1GB -query.max-memory-per-node=512MB -query.max-total-memory-per-node=1GB +query.max-memory-per-node=1GB discovery.uri=https://presto-master:8443 # LDAP specific properties diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2-http-proxy/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2-http-proxy/config.properties index d9dbd3a8f6aa..977ad331c115 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2-http-proxy/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2-http-proxy/config.properties @@ -5,8 +5,7 @@ coordinator=true node-scheduler.include-coordinator=true http-server.http.port=8080 query.max-memory=2GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=1.25GB +query.max-memory-per-node=1.25GB discovery.uri=http://presto-master:8080 web-ui.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2-https-proxy/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2-https-proxy/config.properties index 8fd73aee2a8d..c637bd72bb92 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2-https-proxy/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2-https-proxy/config.properties @@ -5,8 +5,7 @@ coordinator=true node-scheduler.include-coordinator=true http-server.http.port=8080 query.max-memory=2GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=1.25GB +query.max-memory-per-node=1.25GB discovery.uri=http://presto-master:8080 web-ui.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2/config.properties index bd74c372c37f..120c6068b157 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-oauth2/config.properties @@ -5,8 +5,7 @@ coordinator=true node-scheduler.include-coordinator=true http-server.http.port=8080 query.max-memory=2GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=1.25GB +query.max-memory-per-node=1.25GB discovery.uri=http://presto-master:8080 web-ui.enabled=true diff --git a/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java b/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java index 8e97ad2e31e1..5d85106cb086 100644 --- a/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java +++ b/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java @@ -686,7 +686,6 @@ public void testOutputStats() assertEquals(statistics.getPlanningTime().get().toMillis(), queryStats.getPlanningTime().toMillis()); assertEquals(statistics.getExecutionTime().get().toMillis(), queryStats.getExecutionTime().toMillis()); assertEquals(statistics.getPeakUserMemoryBytes(), queryStats.getPeakUserMemoryReservation().toBytes()); - assertEquals(statistics.getPeakTotalNonRevocableMemoryBytes(), queryStats.getPeakNonRevocableMemoryReservation().toBytes()); assertEquals(statistics.getPeakTaskUserMemory(), queryStats.getPeakTaskUserMemory().toBytes()); assertEquals(statistics.getPeakTaskTotalMemory(), queryStats.getPeakTaskTotalMemory().toBytes()); assertEquals(statistics.getPhysicalInputBytes(), queryStats.getPhysicalInputDataSize().toBytes()); diff --git a/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java b/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java index 9e65c8b0f6fd..b294ff948647 100644 --- a/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java +++ b/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java @@ -93,7 +93,6 @@ private static BasicQueryInfo createQueryInfo(String queryId, QueryState state) 22, DataSize.valueOf("20GB"), 23, - 11, DataSize.valueOf("23GB"), DataSize.valueOf("24GB"), DataSize.valueOf("25GB"), diff --git a/testing/trino-tests/src/test/java/io/trino/memory/TestMemoryManager.java b/testing/trino-tests/src/test/java/io/trino/memory/TestMemoryManager.java index 05f24233867f..6ab0a2161450 100644 --- a/testing/trino-tests/src/test/java/io/trino/memory/TestMemoryManager.java +++ b/testing/trino-tests/src/test/java/io/trino/memory/TestMemoryManager.java @@ -27,6 +27,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -87,14 +88,13 @@ public void testResourceOverCommit() { Map properties = ImmutableMap.builder() .put("query.max-memory-per-node", "1kB") - .put("query.max-total-memory-per-node", "1kB") .put("query.max-memory", "1kB") .buildOrThrow(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { assertThatThrownBy(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")) .isInstanceOf(RuntimeException.class) - .hasMessageStartingWith("Query exceeded per-node total memory limit of "); + .hasMessageStartingWith("Query exceeded per-node memory limit of "); Session session = testSessionBuilder() .setCatalog("tpch") .setSchema("tiny") @@ -359,20 +359,26 @@ public void testQueryUserMemoryLimit() } } - @Test(timeOut = 60_000, expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Query exceeded distributed total memory limit of 2kB.*") + @Test(timeOut = 60_000, expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Query exceeded distributed total memory limit of 120MB.*") public void testQueryTotalMemoryLimit() throws Exception { Map properties = ImmutableMap.builder() - .put("query.max-memory", "1kB") - .put("query.max-total-memory", "2kB") + // Relatively high memory limit is required, so that the table scan memory usage alone does not cause the query to fail. + .put("query.max-memory", "120MB") + .put("query.max-total-memory", "120MB") + // The user memory enforcement is tested in testQueryTotalMemoryLimit(). + // Total memory = user memory + revocable memory. + .put("spill-enabled", "true") + .put("spiller-spill-path", Paths.get(System.getProperty("java.io.tmpdir"), "trino", "spills").toString()) + .put("spiller-max-used-space-threshold", "1.0") .buildOrThrow(); try (QueryRunner queryRunner = createQueryRunner(SESSION, properties)) { - queryRunner.execute(SESSION, "SELECT COUNT(*), repeat(orderstatus, 1000) FROM orders GROUP BY 2"); + queryRunner.execute(SESSION, "SELECT * FROM tpch.sf10.orders ORDER BY orderkey"); } } - @Test(timeOut = 60_000, expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Query exceeded per-node user memory limit of 1kB.*") + @Test(timeOut = 60_000, expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Query exceeded per-node memory limit of 1kB.*") public void testQueryMemoryPerNodeLimit() throws Exception { diff --git a/testing/trino-tests/src/test/java/io/trino/memory/TestMemorySessionProperties.java b/testing/trino-tests/src/test/java/io/trino/memory/TestMemorySessionProperties.java index a6154906ac37..ab5094a5e97e 100644 --- a/testing/trino-tests/src/test/java/io/trino/memory/TestMemorySessionProperties.java +++ b/testing/trino-tests/src/test/java/io/trino/memory/TestMemorySessionProperties.java @@ -20,7 +20,6 @@ import org.testng.annotations.Test; import static io.trino.SystemSessionProperties.QUERY_MAX_MEMORY_PER_NODE; -import static io.trino.SystemSessionProperties.QUERY_MAX_TOTAL_MEMORY_PER_NODE; import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestMemorySessionProperties @@ -44,18 +43,6 @@ public void testSessionQueryMemoryPerNodeLimit() .build(); assertThatThrownBy(() -> getQueryRunner().execute(session, sql)) .isInstanceOf(RuntimeException.class) - .hasMessageStartingWith("Query exceeded per-node user memory limit of "); - } - - @Test(timeOut = 240_000) - public void testSessionQueryMaxTotalMemoryPerNodeLimit() - { - assertQuery(sql); - Session session = Session.builder(getSession()) - .setSystemProperty(QUERY_MAX_TOTAL_MEMORY_PER_NODE, "1kB") - .build(); - assertThatThrownBy(() -> getQueryRunner().execute(session, sql)) - .isInstanceOf(RuntimeException.class) - .hasMessageStartingWith("Query exceeded per-node total memory limit of "); + .hasMessageStartingWith("Query exceeded per-node memory limit of "); } }