diff --git a/presto-main/src/main/java/com/facebook/presto/execution/BasicStageExecutionStats.java b/presto-main/src/main/java/com/facebook/presto/execution/BasicStageExecutionStats.java index 063ff8afb942f..20657fd8cdd89 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/BasicStageExecutionStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/BasicStageExecutionStats.java @@ -42,7 +42,7 @@ public class BasicStageExecutionStats new DataSize(0, BYTE), 0, - 0, + 0.0, new DataSize(0, BYTE), new DataSize(0, BYTE), @@ -63,7 +63,7 @@ public class BasicStageExecutionStats private final int completedDrivers; private final DataSize rawInputDataSize; private final long rawInputPositions; - private final long cumulativeUserMemory; + private final double cumulativeUserMemory; private final DataSize userMemoryReservation; private final DataSize totalMemoryReservation; private final Duration totalCpuTime; @@ -84,7 +84,7 @@ public BasicStageExecutionStats( DataSize rawInputDataSize, long rawInputPositions, - long cumulativeUserMemory, + double cumulativeUserMemory, DataSize userMemoryReservation, DataSize totalMemoryReservation, @@ -151,7 +151,7 @@ public long getRawInputPositions() return rawInputPositions; } - public long getCumulativeUserMemory() + public double getCumulativeUserMemory() { return cumulativeUserMemory; } @@ -203,7 +203,7 @@ public static BasicStageExecutionStats aggregateBasicStageStats(Iterable> int runningDrivers = 0; int completedDrivers = 0; - long cumulativeUserMemory = 0; + double cumulativeUserMemory = 0; long userMemoryReservation = 0; long totalMemoryReservation = 0; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java index 8fd01bf1e24ae..b16993807d84c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java @@ -422,7 +422,7 @@ public BasicStageExecutionStats toBasicStageStats(StageExecutionState stageExecu completedDrivers, rawInputDataSize, rawInputPositions, - (long) cumulativeUserMemory, + cumulativeUserMemory, userMemoryReservation, totalMemoryReservation, totalCpuTime, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java b/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java index 96663dfbb847a..3bb85e2d35e70 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java @@ -503,6 +503,9 @@ public TaskStats getTaskStats() updatePeakMemory(); synchronized (cumulativeMemoryLock) { + if (lastTaskStatCallNanos == 0) { + lastTaskStatCallNanos = startNanos; + } double sinceLastPeriodMillis = (System.nanoTime() - lastTaskStatCallNanos) / 1_000_000.0; long averageMemoryForLastPeriod = (userMemory + lastUserMemoryReservation) / 2; cumulativeUserMemory.addAndGet(averageMemoryForLastPeriod * sinceLastPeriodMillis); diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryTracking.java b/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryTracking.java index 24596a37dad82..92542529087a1 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryTracking.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryTracking.java @@ -374,6 +374,22 @@ public void testDestroy() assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 0, 0); } + @Test + public void testCumulativeUserMemoryEstimation() + { + LocalMemoryContext userMemory = operatorContext.localUserMemoryContext(); + long userMemoryBytes = 100_000_000; + userMemory.setBytes(userMemoryBytes); + long startTime = System.nanoTime(); + double cumulativeUserMemory = taskContext.getTaskStats().getCumulativeUserMemory(); + long endTime = System.nanoTime(); + + double elapsedTimeInMillis = (endTime - startTime) / 1_000_000.0; + long averageMemoryForLastPeriod = userMemoryBytes / 2; + + assertTrue(cumulativeUserMemory < elapsedTimeInMillis * averageMemoryForLastPeriod); + } + private void assertStats( OperatorStats operatorStats, DriverStats driverStats,