diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/QuerySystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/QuerySystemTable.java index 87077b3143f06..dee2de8039720 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/QuerySystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/QuerySystemTable.java @@ -31,7 +31,6 @@ import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.inject.Inject; @@ -44,6 +43,7 @@ import static com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType; import static com.facebook.presto.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; import static com.facebook.presto.spi.SystemTable.Distribution.ALL_COORDINATORS; +import static com.facebook.presto.util.DateTimeUtils.toTimeStampInMillis; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.Slices.utf8Slice; import static java.util.Objects.requireNonNull; @@ -119,10 +119,10 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect toMillis(queryStats.getQueuedTime()), toMillis(queryStats.getAnalysisTime()), - toTimeStamp(queryStats.getCreateTime()), - toTimeStamp(queryStats.getExecutionStartTime()), - toTimeStamp(queryStats.getLastHeartbeat()), - toTimeStamp(queryStats.getEndTime())); + toTimeStampInMillis(queryStats.getCreateTime()), + toTimeStampInMillis(queryStats.getExecutionStartTime()), + toTimeStampInMillis(queryStats.getLastHeartbeat()), + toTimeStampInMillis(queryStats.getEndTime())); } return table.build().cursor(); } @@ -145,12 +145,4 @@ private static Long toMillis(Duration duration) } return duration.toMillis(); } - - private static Long toTimeStamp(DateTime dateTime) - { - if (dateTime == null) { - return null; - } - return dateTime.getMillis(); - } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/TaskSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/TaskSystemTable.java index bba4922b326e8..6d50a6444b370 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/TaskSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/TaskSystemTable.java @@ -29,7 +29,6 @@ import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import io.airlift.units.DataSize; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.inject.Inject; @@ -138,10 +137,10 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect stats.getPhysicalWrittenDataSizeInBytes(), - toTimeStamp(stats.getCreateTime()), - toTimeStamp(stats.getFirstStartTime()), - toTimeStamp(taskInfo.getLastHeartbeat()), - toTimeStamp(stats.getEndTime())); + stats.getCreateTimeInMillis(), + stats.getFirstStartTimeInMillis(), + taskInfo.getLastHeartbeatInMillis(), + stats.getEndTimeInMillis()); } return table.build().cursor(); } @@ -161,12 +160,4 @@ private static Long toBytes(DataSize dataSize) } return dataSize.toBytes(); } - - private static Long toTimeStamp(DateTime dateTime) - { - if (dateTime == null) { - return null; - } - return dateTime.getMillis(); - } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/TransactionsSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/TransactionsSystemTable.java index 5a0977baa47e2..83140c0ec7ff1 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/TransactionsSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/TransactionsSystemTable.java @@ -92,7 +92,7 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect info.getIsolationLevel().toString(), info.isReadOnly(), info.isAutoCommitContext(), - info.getCreateTime().getMillis(), + info.getCreateTimeInMillis(), (long) info.getIdleTime().getValue(TimeUnit.SECONDS), info.getWrittenConnectorId().map(ConnectorId::getCatalogName).orElse(null), createStringsBlock(info.getConnectorIds())); diff --git a/presto-main/src/main/java/com/facebook/presto/dispatcher/FailedDispatchQuery.java b/presto-main/src/main/java/com/facebook/presto/dispatcher/FailedDispatchQuery.java index 1d75101f93c54..b3dbc66fd6f49 100644 --- a/presto-main/src/main/java/com/facebook/presto/dispatcher/FailedDispatchQuery.java +++ b/presto-main/src/main/java/com/facebook/presto/dispatcher/FailedDispatchQuery.java @@ -24,7 +24,6 @@ import com.facebook.presto.spi.resourceGroups.ResourceGroupQueryLimits; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.Duration; -import org.joda.time.DateTime; import java.net.URI; import java.util.Optional; @@ -146,27 +145,27 @@ public boolean isRetry() public void recordHeartbeat() {} @Override - public DateTime getLastHeartbeat() + public long getLastHeartbeatInMillis() { - return basicQueryInfo.getQueryStats().getEndTime(); + return basicQueryInfo.getQueryStats().getEndTimeInMillis(); } @Override - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return basicQueryInfo.getQueryStats().getCreateTime(); + return basicQueryInfo.getQueryStats().getCreateTimeInMillis(); } @Override - public Optional getExecutionStartTime() + public long getExecutionStartTimeInMillis() { - return getEndTime(); + return getEndTimeInMillis(); } @Override - public Optional getEndTime() + public long getEndTimeInMillis() { - return Optional.ofNullable(basicQueryInfo.getQueryStats().getEndTime()); + return basicQueryInfo.getQueryStats().getEndTimeInMillis(); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQuery.java b/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQuery.java index dfe7d1b3c21c2..eb65385df75fa 100644 --- a/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQuery.java +++ b/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQuery.java @@ -35,7 +35,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import io.airlift.units.Duration; -import org.joda.time.DateTime; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -230,9 +229,9 @@ public void recordHeartbeat() } @Override - public DateTime getLastHeartbeat() + public long getLastHeartbeatInMillis() { - return stateMachine.getLastHeartbeat(); + return stateMachine.getLastHeartbeatInMillis(); } @Override @@ -275,21 +274,21 @@ public boolean isDone() } @Override - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return stateMachine.getCreateTime(); + return stateMachine.getCreateTimeInMillis(); } @Override - public Optional getExecutionStartTime() + public long getExecutionStartTimeInMillis() { - return stateMachine.getExecutionStartTime(); + return stateMachine.getExecutionStartTimeInMillis(); } @Override - public Optional getEndTime() + public long getEndTimeInMillis() { - return stateMachine.getEndTime(); + return stateMachine.getEndTimeInMillis(); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java index 3f2712a2b5cb2..c60d67de3cd6b 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java @@ -74,7 +74,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.joda.time.DateTime; import javax.inject.Inject; @@ -660,11 +659,11 @@ private static void logQueryTimeline(QueryInfo queryInfo) { try { QueryStats queryStats = queryInfo.getQueryStats(); - DateTime queryStartTime = queryStats.getCreateTime(); - DateTime queryEndTime = queryStats.getEndTime(); + long queryStartTime = queryStats.getCreateTimeInMillis(); + long queryEndTime = queryStats.getEndTimeInMillis(); // query didn't finish cleanly - if (queryStartTime == null || queryEndTime == null) { + if (queryStartTime == 0 || queryEndTime == 0) { return; } @@ -673,9 +672,9 @@ private static void logQueryTimeline(QueryInfo queryInfo) List stages = getAllStages(queryInfo.getOutputStage()); // long lastSchedulingCompletion = 0; - long firstTaskStartTime = queryEndTime.getMillis(); - long lastTaskStartTime = queryStartTime.getMillis() + planning; - long lastTaskEndTime = queryStartTime.getMillis() + planning; + long firstTaskStartTime = queryEndTime; + long lastTaskStartTime = queryStartTime + planning; + long lastTaskEndTime = queryStartTime + planning; for (StageInfo stage : stages) { // only consider leaf stages if (!stage.getSubStages().isEmpty()) { @@ -685,27 +684,27 @@ private static void logQueryTimeline(QueryInfo queryInfo) for (TaskInfo taskInfo : stage.getLatestAttemptExecutionInfo().getTasks()) { TaskStats taskStats = taskInfo.getStats(); - DateTime firstStartTime = taskStats.getFirstStartTime(); - if (firstStartTime != null) { - firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime); + long firstStartTimeInMillis = taskStats.getFirstStartTimeInMillis(); + if (firstStartTimeInMillis != 0) { + firstTaskStartTime = Math.min(firstStartTimeInMillis, firstTaskStartTime); } - DateTime lastStartTime = taskStats.getLastStartTime(); - if (lastStartTime != null) { - lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime); + long lastStartTimeInMillis = taskStats.getLastStartTimeInMillis(); + if (lastStartTimeInMillis != 0) { + lastTaskStartTime = max(lastStartTimeInMillis, lastTaskStartTime); } - DateTime endTime = taskStats.getEndTime(); - if (endTime != null) { - lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime); + long endTimeInMillis = taskStats.getEndTimeInMillis(); + if (endTimeInMillis != 0) { + lastTaskEndTime = max(endTimeInMillis, lastTaskEndTime); } } } - long elapsed = max(queryEndTime.getMillis() - queryStartTime.getMillis(), 0); - long scheduling = max(firstTaskStartTime - queryStartTime.getMillis() - planning, 0); + long elapsed = max(queryEndTime - queryStartTime, 0); + long scheduling = max(firstTaskStartTime - queryStartTime - planning, 0); long running = max(lastTaskEndTime - firstTaskStartTime, 0); - long finishing = max(queryEndTime.getMillis() - lastTaskEndTime, 0); + long finishing = max(queryEndTime - lastTaskEndTime, 0); logQueryTimeline( queryInfo.getQueryId(), @@ -725,15 +724,15 @@ private static void logQueryTimeline(QueryInfo queryInfo) private static void logQueryTimeline(BasicQueryInfo queryInfo) { - DateTime queryStartTime = queryInfo.getQueryStats().getCreateTime(); - DateTime queryEndTime = queryInfo.getQueryStats().getEndTime(); + long queryStartTimeInMillis = queryInfo.getQueryStats().getCreateTimeInMillis(); + long queryEndTimeInMillis = queryInfo.getQueryStats().getEndTimeInMillis(); // query didn't finish cleanly - if (queryStartTime == null || queryEndTime == null) { + if (queryStartTimeInMillis == 0 || queryEndTimeInMillis == 0) { return; } - long elapsed = max(queryEndTime.getMillis() - queryStartTime.getMillis(), 0); + long elapsed = max(queryEndTimeInMillis - queryStartTimeInMillis, 0); logQueryTimeline( queryInfo.getQueryId(), @@ -743,8 +742,8 @@ private static void logQueryTimeline(BasicQueryInfo queryInfo) 0, 0, 0, - queryStartTime, - queryEndTime); + queryStartTimeInMillis, + queryEndTimeInMillis); } private static void logQueryTimeline( @@ -755,10 +754,10 @@ private static void logQueryTimeline( long schedulingMillis, long runningMillis, long finishingMillis, - DateTime queryStartTime, - DateTime queryEndTime) + long queryStartTimeInMillis, + long queryEndTimeInMillis) { - log.info("TIMELINE: Query %s :: Transaction:[%s] :: elapsed %sms :: planning %sms :: scheduling %sms :: running %sms :: finishing %sms :: begin %s :: end %s", + log.info("TIMELINE: Query %s :: Transaction:[%s] :: elapsed %sms :: planning %sms :: scheduling %sms :: running %sms :: finishing %sms :: begin %sms :: end %sms", queryId, transactionId, elapsedMillis, @@ -766,8 +765,8 @@ private static void logQueryTimeline( schedulingMillis, runningMillis, finishingMillis, - queryStartTime, - queryEndTime); + queryStartTimeInMillis, + queryEndTimeInMillis); } private static ResourceDistribution createResourceDistribution( diff --git a/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java index 5c425d6cf4e12..10db412e3bd72 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java @@ -30,6 +30,7 @@ import java.util.Optional; import static java.time.Duration.ofMillis; +import static java.time.Instant.ofEpochMilli; import static java.util.Objects.requireNonNull; public class SplitMonitor @@ -59,13 +60,13 @@ public void splitFailedEvent(TaskId taskId, DriverStats driverStats, Throwable c private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullable String failureType, @Nullable String failureMessage) { Optional timeToStart = Optional.empty(); - if (driverStats.getStartTime() != null) { - timeToStart = Optional.of(ofMillis(driverStats.getStartTime().getMillis() - driverStats.getCreateTime().getMillis())); + if (driverStats.getStartTimeInMillis() != 0) { + timeToStart = Optional.of(ofMillis(driverStats.getStartTimeInMillis() - driverStats.getCreateTimeInMillis())); } Optional timeToEnd = Optional.empty(); - if (driverStats.getEndTime() != null) { - timeToEnd = Optional.of(ofMillis(driverStats.getEndTime().getMillis() - driverStats.getCreateTime().getMillis())); + if (driverStats.getEndTimeInMillis() != 0) { + timeToEnd = Optional.of(ofMillis(driverStats.getEndTimeInMillis() - driverStats.getCreateTimeInMillis())); } Optional splitFailureMetadata = Optional.empty(); @@ -80,9 +81,9 @@ private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullab taskId.getStageExecutionId().getStageId().toString(), taskId.getStageExecutionId().toString(), Integer.toString(taskId.getId()), - driverStats.getCreateTime().toDate().toInstant(), - Optional.ofNullable(driverStats.getStartTime()).map(startTime -> startTime.toDate().toInstant()), - Optional.ofNullable(driverStats.getEndTime()).map(endTime -> endTime.toDate().toInstant()), + ofEpochMilli(driverStats.getCreateTimeInMillis()), + Optional.ofNullable(ofEpochMilli(driverStats.getStartTimeInMillis())), + Optional.ofNullable(ofEpochMilli(driverStats.getEndTimeInMillis())), new SplitStatistics( ofMillis(driverStats.getTotalCpuTime().toMillis()), ofMillis(driverStats.getElapsedTime().toMillis()), diff --git a/presto-main/src/main/java/com/facebook/presto/execution/AccessControlCheckerExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/AccessControlCheckerExecution.java index 3e96b9566f151..4389045c1ee39 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/AccessControlCheckerExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/AccessControlCheckerExecution.java @@ -37,7 +37,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -138,27 +137,27 @@ public long getTotalMemoryReservationInBytes() } @Override - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return stateMachine.getCreateTime(); + return stateMachine.getCreateTimeInMillis(); } @Override - public Optional getExecutionStartTime() + public long getExecutionStartTimeInMillis() { - return stateMachine.getExecutionStartTime(); + return stateMachine.getExecutionStartTimeInMillis(); } @Override - public DateTime getLastHeartbeat() + public long getLastHeartbeatInMillis() { - return stateMachine.getLastHeartbeat(); + return stateMachine.getLastHeartbeatInMillis(); } @Override - public Optional getEndTime() + public long getEndTimeInMillis() { - return stateMachine.getEndTime(); + return stateMachine.getEndTimeInMillis(); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/execution/DataDefinitionExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/DataDefinitionExecution.java index a5c43256a4acd..49c22ac63e363 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/DataDefinitionExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/DataDefinitionExecution.java @@ -29,7 +29,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -120,27 +119,27 @@ public long getTotalMemoryReservationInBytes() } @Override - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return stateMachine.getCreateTime(); + return stateMachine.getCreateTimeInMillis(); } @Override - public Optional getExecutionStartTime() + public long getExecutionStartTimeInMillis() { - return stateMachine.getExecutionStartTime(); + return stateMachine.getExecutionStartTimeInMillis(); } @Override - public DateTime getLastHeartbeat() + public long getLastHeartbeatInMillis() { - return stateMachine.getLastHeartbeat(); + return stateMachine.getLastHeartbeatInMillis(); } @Override - public Optional getEndTime() + public long getEndTimeInMillis() { - return stateMachine.getEndTime(); + return stateMachine.getEndTimeInMillis(); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java index 20d267dbee288..010489c76b14e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java @@ -56,7 +56,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import org.joda.time.DateTime; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; @@ -373,8 +372,8 @@ public BasicQueryInfo getBasicQueryInfo(Optional rootS BasicStageExecutionStats stageStats = rootStage.orElse(EMPTY_STAGE_STATS); BasicQueryStats queryStats = new BasicQueryStats( - queryStateTimer.getCreateTime(), - getEndTime().orElse(null), + queryStateTimer.getCreateTimeInMillis(), + getEndTimeInMillis(), queryStateTimer.getWaitingForPrerequisitesTime(), queryStateTimer.getQueuedTime(), queryStateTimer.getElapsedTime(), @@ -1009,24 +1008,24 @@ public void endAnalysis() queryStateTimer.endAnalysis(); } - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return queryStateTimer.getCreateTime(); + return queryStateTimer.getCreateTimeInMillis(); } - public Optional getExecutionStartTime() + public long getExecutionStartTimeInMillis() { - return queryStateTimer.getExecutionStartTime(); + return queryStateTimer.getExecutionStartTimeInMillis(); } - public DateTime getLastHeartbeat() + public long getLastHeartbeatInMillis() { - return queryStateTimer.getLastHeartbeat(); + return queryStateTimer.getLastHeartbeatInMillis(); } - public Optional getEndTime() + public long getEndTimeInMillis() { - return queryStateTimer.getEndTime(); + return queryStateTimer.getEndTimeInMillis(); } public Optional getFailureInfo() @@ -1278,10 +1277,10 @@ private static StageExecutionInfo pruneStageExecutionInfo(StageExecutionInfo inf private static QueryStats pruneQueryStats(QueryStats queryStats) { return new QueryStats( - queryStats.getCreateTime(), - queryStats.getExecutionStartTime(), - queryStats.getLastHeartbeat(), - queryStats.getEndTime(), + queryStats.getCreateTimeInMillis(), + queryStats.getExecutionStartTimeInMillis(), + queryStats.getLastHeartbeatInMillis(), + queryStats.getEndTimeInMillis(), queryStats.getElapsedTime(), queryStats.getWaitingForPrerequisitesTime(), queryStats.getQueuedTime(), diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryStateTimer.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryStateTimer.java index c3dc7e50cd587..36da1ab1d57ad 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryStateTimer.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryStateTimer.java @@ -15,9 +15,7 @@ import com.google.common.base.Ticker; import io.airlift.units.Duration; -import org.joda.time.DateTime; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import static io.airlift.units.Duration.succinctNanos; @@ -64,7 +62,7 @@ public class QueryStateTimer { private final Ticker ticker; - private final DateTime createTime = DateTime.now(); + private final long createTimeInMillis = System.currentTimeMillis(); private final long createNanos; private final AtomicReference beginQueuedNanos = new AtomicReference<>(); @@ -250,14 +248,14 @@ public void recordHeartbeat() // Stats // - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return createTime; + return createTimeInMillis; } - public Optional getExecutionStartTime() + public long getExecutionStartTimeInMillis() { - return toDateTime(beginPlanningNanos); + return toMillis(beginPlanningNanos); } public Duration getElapsedTime() @@ -319,9 +317,9 @@ public Duration getExecutionTime() return getDuration(executionTime, beginPlanningNanos); } - public Optional getEndTime() + public long getEndTimeInMillis() { - return toDateTime(endNanos); + return toMillis(endNanos); } public Duration getAnalysisTime() @@ -329,9 +327,9 @@ public Duration getAnalysisTime() return getDuration(analysisTime, beginAnalysisNanos); } - public DateTime getLastHeartbeat() + public long getLastHeartbeatInMillis() { - return toDateTime(lastHeartbeatNanos.get()); + return toMillis(lastHeartbeatNanos); } // @@ -370,18 +368,9 @@ private Duration getDuration(AtomicReference finalDuration, AtomicRefe return new Duration(0, MILLISECONDS); } - private Optional toDateTime(AtomicReference instantNanos) + private long toMillis(AtomicReference instantNanos) { Long nanos = instantNanos.get(); - if (nanos == null) { - return Optional.empty(); - } - return Optional.of(toDateTime(nanos)); - } - - private DateTime toDateTime(long instantNanos) - { - long millisSinceCreate = NANOSECONDS.toMillis(instantNanos - createNanos); - return new DateTime(createTime.getMillis() + millisSinceCreate); + return nanos != null ? createTimeInMillis + NANOSECONDS.toMillis(nanos - createNanos) : 0L; } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java index 26f9f70718f2c..59eb42db30f35 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java @@ -39,6 +39,7 @@ import java.util.OptionalDouble; import java.util.Set; +import static com.facebook.presto.util.DateTimeUtils.toTimeStampInMillis; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.units.DataSize.succinctBytes; import static io.airlift.units.Duration.succinctDuration; @@ -48,11 +49,11 @@ public class QueryStats { - private final DateTime createTime; + private final long createTimeInMillis; - private final DateTime executionStartTime; - private final DateTime lastHeartbeat; - private final DateTime endTime; + private final long executionStartTimeInMillis; + private final long lastHeartbeatInMillis; + private final long endTimeInMillis; private final Duration elapsedTime; private final Duration waitingForPrerequisitesTime; @@ -122,84 +123,85 @@ public class QueryStats // RuntimeStats aggregated at the query level including the metrics exposed in every task and every operator. private final RuntimeStats runtimeStats; - @JsonCreator public QueryStats( - @JsonProperty("createTime") DateTime createTime, - @JsonProperty("executionStartTime") DateTime executionStartTime, - @JsonProperty("lastHeartbeat") DateTime lastHeartbeat, - @JsonProperty("endTime") DateTime endTime, + long createTimeInMillis, + long executionStartTimeInMillis, + long lastHeartbeatInMillis, + long endTimeInMillis, + + Duration elapsedTime, + Duration waitingForPrerequisitesTime, + Duration queuedTime, + Duration resourceWaitingTime, + Duration semanticAnalyzingTime, + Duration columnAccessPermissionCheckingTime, + Duration dispatchingTime, + Duration executionTime, + Duration analysisTime, + Duration totalPlanningTime, + Duration finishingTime, + + int totalTasks, + int runningTasks, + int peakRunningTasks, + int completedTasks, - @JsonProperty("elapsedTime") Duration elapsedTime, - @JsonProperty("waitingForPrerequisitesTime") Duration waitingForPrerequisitesTime, - @JsonProperty("queuedTime") Duration queuedTime, - @JsonProperty("resourceWaitingTime") Duration resourceWaitingTime, - @JsonProperty("semanticAnalyzingTime") Duration semanticAnalyzingTime, - @JsonProperty("columnAccessPermissionCheckingTime") Duration columnAccessPermissionCheckingTime, - @JsonProperty("dispatchingTime") Duration dispatchingTime, - @JsonProperty("executionTime") Duration executionTime, - @JsonProperty("analysisTime") Duration analysisTime, - @JsonProperty("totalPlanningTime") Duration totalPlanningTime, - @JsonProperty("finishingTime") Duration finishingTime, + int totalDrivers, + int queuedDrivers, + int runningDrivers, + int blockedDrivers, + int completedDrivers, - @JsonProperty("totalTasks") int totalTasks, - @JsonProperty("runningTasks") int runningTasks, - @JsonProperty("peakRunningTasks") int peakRunningTasks, - @JsonProperty("completedTasks") int completedTasks, + double cumulativeUserMemory, + double cumulativeTotalMemory, + DataSize userMemoryReservation, + DataSize totalMemoryReservation, + DataSize peakUserMemoryReservation, + DataSize peakTotalMemoryReservation, + DataSize peakTaskUserMemory, + DataSize peakTaskTotalMemory, + DataSize peakNodeTotalMemory, - @JsonProperty("totalDrivers") int totalDrivers, - @JsonProperty("queuedDrivers") int queuedDrivers, - @JsonProperty("runningDrivers") int runningDrivers, - @JsonProperty("blockedDrivers") int blockedDrivers, - @JsonProperty("completedDrivers") int completedDrivers, - - @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, - @JsonProperty("cumulativeTotalMemory") double cumulativeTotalMemory, - @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, - @JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation, - @JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation, - @JsonProperty("peakTotalMemoryReservation") DataSize peakTotalMemoryReservation, - @JsonProperty("peakTaskUserMemory") DataSize peakTaskUserMemory, - @JsonProperty("peakTaskTotalMemory") DataSize peakTaskTotalMemory, - @JsonProperty("peakNodeTotalMemory") DataSize peakNodeTotalMemory, + boolean scheduled, + Duration totalScheduledTime, + Duration totalCpuTime, + Duration retriedCpuTime, + Duration totalBlockedTime, + boolean fullyBlocked, + Set blockedReasons, - @JsonProperty("scheduled") boolean scheduled, - @JsonProperty("totalScheduledTime") Duration totalScheduledTime, - @JsonProperty("totalCpuTime") Duration totalCpuTime, - @JsonProperty("retriedCpuTime") Duration retriedCpuTime, - @JsonProperty("totalBlockedTime") Duration totalBlockedTime, - @JsonProperty("fullyBlocked") boolean fullyBlocked, - @JsonProperty("blockedReasons") Set blockedReasons, - - @JsonProperty("totalAllocation") DataSize totalAllocation, + DataSize totalAllocation, - @JsonProperty("rawInputDataSize") DataSize rawInputDataSize, - @JsonProperty("rawInputPositions") long rawInputPositions, + DataSize rawInputDataSize, + long rawInputPositions, - @JsonProperty("processedInputDataSize") DataSize processedInputDataSize, - @JsonProperty("processedInputPositions") long processedInputPositions, + DataSize processedInputDataSize, + long processedInputPositions, - @JsonProperty("shuffledDataSize") DataSize shuffledDataSize, - @JsonProperty("shuffledPositions") long shuffledPositions, + DataSize shuffledDataSize, + long shuffledPositions, - @JsonProperty("outputDataSize") DataSize outputDataSize, - @JsonProperty("outputPositions") long outputPositions, + DataSize outputDataSize, + long outputPositions, - @JsonProperty("writtenOutputPositions") long writtenOutputPositions, - @JsonProperty("writtenOutputLogicalDataSize") DataSize writtenOutputLogicalDataSize, - @JsonProperty("writtenOutputPhysicalDataSize") DataSize writtenOutputPhysicalDataSize, + long writtenOutputPositions, + DataSize writtenOutputLogicalDataSize, + DataSize writtenOutputPhysicalDataSize, - @JsonProperty("writtenIntermediatePhysicalDataSize") DataSize writtenIntermediatePhysicalDataSize, + DataSize writtenIntermediatePhysicalDataSize, - @JsonProperty("stageGcStatistics") List stageGcStatistics, + List stageGcStatistics, - @JsonProperty("operatorSummaries") List operatorSummaries, + List operatorSummaries, - @JsonProperty("runtimeStats") RuntimeStats runtimeStats) + RuntimeStats runtimeStats) { - this.createTime = requireNonNull(createTime, "createTime is null"); - this.executionStartTime = executionStartTime; - this.lastHeartbeat = requireNonNull(lastHeartbeat, "lastHeartbeat is null"); - this.endTime = endTime; + checkArgument(createTimeInMillis >= 0, "createTimeInMillis is negative"); + this.createTimeInMillis = createTimeInMillis; + this.executionStartTimeInMillis = executionStartTimeInMillis; + checkArgument(lastHeartbeatInMillis >= 0, "lastHeartbeatInMillis is negative"); + this.lastHeartbeatInMillis = lastHeartbeatInMillis; + this.endTimeInMillis = endTimeInMillis; this.elapsedTime = requireNonNull(elapsedTime, "elapsedTime is null"); this.waitingForPrerequisitesTime = requireNonNull(waitingForPrerequisitesTime, "waitingForPrerequisitesTime is null"); @@ -282,6 +284,152 @@ public QueryStats( this.runtimeStats = (runtimeStats == null) ? new RuntimeStats() : runtimeStats; } + @JsonCreator + public QueryStats( + @JsonProperty("createTime") DateTime createTime, + @JsonProperty("executionStartTime") DateTime executionStartTime, + @JsonProperty("lastHeartbeat") DateTime lastHeartbeat, + @JsonProperty("endTime") DateTime endTime, + + @JsonProperty("elapsedTime") Duration elapsedTime, + @JsonProperty("waitingForPrerequisitesTime") Duration waitingForPrerequisitesTime, + @JsonProperty("queuedTime") Duration queuedTime, + @JsonProperty("resourceWaitingTime") Duration resourceWaitingTime, + @JsonProperty("semanticAnalyzingTime") Duration semanticAnalyzingTime, + @JsonProperty("columnAccessPermissionCheckingTime") Duration columnAccessPermissionCheckingTime, + @JsonProperty("dispatchingTime") Duration dispatchingTime, + @JsonProperty("executionTime") Duration executionTime, + @JsonProperty("analysisTime") Duration analysisTime, + @JsonProperty("totalPlanningTime") Duration totalPlanningTime, + @JsonProperty("finishingTime") Duration finishingTime, + + @JsonProperty("totalTasks") int totalTasks, + @JsonProperty("runningTasks") int runningTasks, + @JsonProperty("peakRunningTasks") int peakRunningTasks, + @JsonProperty("completedTasks") int completedTasks, + + @JsonProperty("totalDrivers") int totalDrivers, + @JsonProperty("queuedDrivers") int queuedDrivers, + @JsonProperty("runningDrivers") int runningDrivers, + @JsonProperty("blockedDrivers") int blockedDrivers, + @JsonProperty("completedDrivers") int completedDrivers, + + @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, + @JsonProperty("cumulativeTotalMemory") double cumulativeTotalMemory, + @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, + @JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation, + @JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation, + @JsonProperty("peakTotalMemoryReservation") DataSize peakTotalMemoryReservation, + @JsonProperty("peakTaskUserMemory") DataSize peakTaskUserMemory, + @JsonProperty("peakTaskTotalMemory") DataSize peakTaskTotalMemory, + @JsonProperty("peakNodeTotalMemory") DataSize peakNodeTotalMemory, + + @JsonProperty("scheduled") boolean scheduled, + @JsonProperty("totalScheduledTime") Duration totalScheduledTime, + @JsonProperty("totalCpuTime") Duration totalCpuTime, + @JsonProperty("retriedCpuTime") Duration retriedCpuTime, + @JsonProperty("totalBlockedTime") Duration totalBlockedTime, + @JsonProperty("fullyBlocked") boolean fullyBlocked, + @JsonProperty("blockedReasons") Set blockedReasons, + + @JsonProperty("totalAllocation") DataSize totalAllocation, + + @JsonProperty("rawInputDataSize") DataSize rawInputDataSize, + @JsonProperty("rawInputPositions") long rawInputPositions, + + @JsonProperty("processedInputDataSize") DataSize processedInputDataSize, + @JsonProperty("processedInputPositions") long processedInputPositions, + + @JsonProperty("shuffledDataSize") DataSize shuffledDataSize, + @JsonProperty("shuffledPositions") long shuffledPositions, + + @JsonProperty("outputDataSize") DataSize outputDataSize, + @JsonProperty("outputPositions") long outputPositions, + + @JsonProperty("writtenOutputPositions") long writtenOutputPositions, + @JsonProperty("writtenOutputLogicalDataSize") DataSize writtenOutputLogicalDataSize, + @JsonProperty("writtenOutputPhysicalDataSize") DataSize writtenOutputPhysicalDataSize, + + @JsonProperty("writtenIntermediatePhysicalDataSize") DataSize writtenIntermediatePhysicalDataSize, + + @JsonProperty("stageGcStatistics") List stageGcStatistics, + + @JsonProperty("operatorSummaries") List operatorSummaries, + + @JsonProperty("runtimeStats") RuntimeStats runtimeStats) + { + this(toTimeStampInMillis(createTime), + toTimeStampInMillis(executionStartTime), + toTimeStampInMillis(lastHeartbeat), + toTimeStampInMillis(endTime), + + elapsedTime, + waitingForPrerequisitesTime, + queuedTime, + resourceWaitingTime, + semanticAnalyzingTime, + columnAccessPermissionCheckingTime, + dispatchingTime, + executionTime, + analysisTime, + totalPlanningTime, + finishingTime, + + totalTasks, + runningTasks, + peakRunningTasks, + completedTasks, + + totalDrivers, + queuedDrivers, + runningDrivers, + blockedDrivers, + completedDrivers, + + cumulativeUserMemory, + cumulativeTotalMemory, + userMemoryReservation, + totalMemoryReservation, + peakUserMemoryReservation, + peakTotalMemoryReservation, + peakTaskUserMemory, + peakTaskTotalMemory, + peakNodeTotalMemory, + + scheduled, + totalScheduledTime, + totalCpuTime, + retriedCpuTime, + totalBlockedTime, + fullyBlocked, + blockedReasons, + + totalAllocation, + + rawInputDataSize, + rawInputPositions, + + processedInputDataSize, + processedInputPositions, + + shuffledDataSize, + shuffledPositions, + + outputDataSize, + outputPositions, + + writtenOutputPositions, + writtenOutputLogicalDataSize, + writtenOutputPhysicalDataSize, + + writtenIntermediatePhysicalDataSize, + + stageGcStatistics, + operatorSummaries, + + runtimeStats); + } + public static QueryStats create( QueryStateTimer queryStateTimer, Optional rootStage, @@ -424,10 +572,10 @@ else if (operatorType.equals(TableScanOperator.class.getSimpleName()) || operato .allMatch(state -> (state == StageExecutionState.RUNNING) || state.isDone()); return new QueryStats( - queryStateTimer.getCreateTime(), - queryStateTimer.getExecutionStartTime().orElse(null), - queryStateTimer.getLastHeartbeat(), - queryStateTimer.getEndTime().orElse(null), + queryStateTimer.getCreateTimeInMillis(), + queryStateTimer.getExecutionStartTimeInMillis(), + queryStateTimer.getLastHeartbeatInMillis(), + queryStateTimer.getEndTimeInMillis(), queryStateTimer.getElapsedTime(), queryStateTimer.getWaitingForPrerequisitesTime(), @@ -505,7 +653,7 @@ private static long computeRetriedCpuTime(StageInfo stageInfo) public static QueryStats immediateFailureQueryStats() { - DateTime now = DateTime.now(); + long now = System.currentTimeMillis(); return new QueryStats( now, now, @@ -568,26 +716,46 @@ public static QueryStats immediateFailureQueryStats() @JsonProperty public DateTime getCreateTime() { - return createTime; + return new DateTime(createTimeInMillis); + } + + public long getCreateTimeInMillis() + { + return createTimeInMillis; } @JsonProperty public DateTime getExecutionStartTime() { - return executionStartTime; + return new DateTime(executionStartTimeInMillis); + } + + public long getExecutionStartTimeInMillis() + { + return executionStartTimeInMillis; } @JsonProperty public DateTime getLastHeartbeat() { - return lastHeartbeat; + return new DateTime(lastHeartbeatInMillis); + } + + public long getLastHeartbeatInMillis() + { + return lastHeartbeatInMillis; } @Nullable @JsonProperty public DateTime getEndTime() { - return endTime; + return new DateTime(endTimeInMillis); + } + + public long getEndTimeInMillis() + { + return endTimeInMillis; } @JsonProperty diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryTracker.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryTracker.java index fe60f77d2196a..b52c07c759eaf 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryTracker.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryTracker.java @@ -23,7 +23,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -54,6 +53,7 @@ import static com.facebook.presto.spi.StandardErrorCode.SERVER_SHUTTING_DOWN; import static com.google.common.base.Preconditions.checkState; import static java.lang.String.format; +import static java.lang.System.currentTimeMillis; import static java.util.Comparator.comparingInt; import static java.util.Objects.requireNonNull; @@ -227,9 +227,9 @@ private void enforceTimeLimits() query.getResourceGroupQueryLimits() .flatMap(ResourceGroupQueryLimits::getExecutionTimeLimit) .map(rgLimit -> createDurationLimit(rgLimit, RESOURCE_GROUP)).orElse(null)); - Optional executionStartTime = query.getExecutionStartTime(); - DateTime createTime = query.getCreateTime(); - if (executionStartTime.isPresent() && executionStartTime.get().plus(queryMaxExecutionTime.getLimit().toMillis()).isBeforeNow()) { + long executionStartTime = query.getExecutionStartTimeInMillis(); + long createTimeInMillis = query.getCreateTimeInMillis(); + if (executionStartTime > 0 && (executionStartTime + queryMaxExecutionTime.getLimit().toMillis()) < currentTimeMillis()) { query.fail( new PrestoException(EXCEEDED_TIME_LIMIT, format( @@ -237,7 +237,7 @@ private void enforceTimeLimits() queryMaxExecutionTime.getLimit(), queryMaxExecutionTime.getLimitSource().name()))); } - if (createTime.plus(queryMaxRunTime.toMillis()).isBeforeNow()) { + if (createTimeInMillis + queryMaxRunTime.toMillis() < currentTimeMillis()) { query.fail(new PrestoException(EXCEEDED_TIME_LIMIT, "Query exceeded maximum time limit of " + queryMaxRunTime)); } } @@ -328,7 +328,7 @@ private void pruneExpiredQueries() */ private void removeExpiredQueries() { - DateTime timeHorizon = DateTime.now().minus(minQueryExpireAge.toMillis()); + long timeHorizonInMillis = currentTimeMillis() - minQueryExpireAge.toMillis(); // we're willing to keep queries beyond timeHorizon as long as we have fewer than maxQueryHistory while (expirationQueue.size() > maxQueryHistory) { @@ -339,12 +339,12 @@ private void removeExpiredQueries() // expirationQueue is FIFO based on query end time. Stop when we see the // first query that's too young to expire - Optional endTime = query.getEndTime(); - if (!endTime.isPresent()) { + long endTimeInMillis = query.getEndTimeInMillis(); + if (endTimeInMillis == 0) { // this shouldn't happen but it is better to be safe here continue; } - if (endTime.get().isAfter(timeHorizon)) { + if (endTimeInMillis > timeHorizonInMillis) { return; } @@ -370,10 +370,10 @@ private void failAbandonedQueries() log.info("Failing abandoned query %s", query.getQueryId()); query.fail(new PrestoException( ABANDONED_QUERY, - format("Query %s has not been accessed since %s: currentTime %s", + format("Query %s has not been accessed since %sms: currentTime %sms", query.getQueryId(), - query.getLastHeartbeat(), - DateTime.now()))); + query.getLastHeartbeatInMillis(), + currentTimeMillis()))); } } catch (RuntimeException e) { @@ -384,10 +384,10 @@ private void failAbandonedQueries() private boolean isAbandoned(T query) { - DateTime oldestAllowedHeartbeat = DateTime.now().minus(clientTimeout.toMillis()); - DateTime lastHeartbeat = query.getLastHeartbeat(); + long oldestAllowedHeartbeatInMillis = currentTimeMillis() - clientTimeout.toMillis(); + long lastHeartbeat = query.getLastHeartbeatInMillis(); - return lastHeartbeat != null && lastHeartbeat.isBefore(oldestAllowedHeartbeat); + return lastHeartbeat > 0 && lastHeartbeat < oldestAllowedHeartbeatInMillis; } public interface TrackedQuery @@ -398,13 +398,13 @@ public interface TrackedQuery Session getSession(); - DateTime getCreateTime(); + long getCreateTimeInMillis(); - Optional getExecutionStartTime(); + long getExecutionStartTimeInMillis(); - DateTime getLastHeartbeat(); + long getLastHeartbeatInMillis(); - Optional getEndTime(); + long getEndTimeInMillis(); Optional getResourceGroupQueryLimits(); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java index 3b541694a6e98..44a39427b0c84 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java @@ -65,7 +65,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.concurrent.ThreadSafe; import javax.inject.Inject; @@ -328,9 +327,9 @@ public long getTotalMemoryReservationInBytes() * Gets the timestamp this query was registered for execution with the query state machine */ @Override - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return stateMachine.getCreateTime(); + return stateMachine.getCreateTimeInMillis(); } /** @@ -338,15 +337,15 @@ public DateTime getCreateTime() * Otherwise returns a {@link Optional#empty()} */ @Override - public Optional getExecutionStartTime() + public long getExecutionStartTimeInMillis() { - return stateMachine.getExecutionStartTime(); + return stateMachine.getExecutionStartTimeInMillis(); } @Override - public DateTime getLastHeartbeat() + public long getLastHeartbeatInMillis() { - return stateMachine.getLastHeartbeat(); + return stateMachine.getLastHeartbeatInMillis(); } /** @@ -354,9 +353,9 @@ public DateTime getLastHeartbeat() * Otherwise returns a {@link Optional#empty()} */ @Override - public Optional getEndTime() + public long getEndTimeInMillis() { - return stateMachine.getEndTime(); + return stateMachine.getEndTimeInMillis(); } /** diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java index 65d1b7901a09b..08036cb7664d9 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java @@ -82,7 +82,7 @@ public class SqlTask private final SqlTaskExecutionFactory sqlTaskExecutionFactory; private final TaskExchangeClientManager taskExchangeClientManager; - private final AtomicReference lastHeartbeat = new AtomicReference<>(DateTime.now()); + private final AtomicLong lastHeartbeat = new AtomicLong(System.currentTimeMillis()); private final AtomicLong nextTaskInfoVersion = new AtomicLong(TaskStatus.STARTING_VERSION); private final AtomicReference taskHolderReference = new AtomicReference<>(new TaskHolder()); @@ -225,7 +225,7 @@ public String getTaskInstanceId() public void recordHeartbeat() { - lastHeartbeat.set(DateTime.now()); + lastHeartbeat.set(System.currentTimeMillis()); } public TaskState getTaskState() @@ -233,9 +233,9 @@ public TaskState getTaskState() return taskStateMachine.getState(); } - public DateTime getTaskCreatedTime() + public long getTaskCreatedTime() { - return taskStateMachine.getCreatedTime(); + return taskStateMachine.getCreatedTimeInMillis(); } public TaskInfo getTaskInfo() @@ -345,8 +345,8 @@ private TaskStats getTaskStats(TaskHolder taskHolder) return taskExecution.getTaskContext().getTaskStats(); } // if the task completed without creation, set end time - DateTime endTime = taskStateMachine.getState().isDone() ? DateTime.now() : null; - return new TaskStats(taskStateMachine.getCreatedTime(), endTime); + long endTimeInMillis = taskStateMachine.getState().isDone() ? System.currentTimeMillis() : 0L; + return new TaskStats(new DateTime(taskStateMachine.getCreatedTimeInMillis()), new DateTime(endTimeInMillis)); } private MetadataUpdates getMetadataUpdateRequests(TaskHolder taskHolder) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java index 226c60baf1f99..443e96adc789e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java @@ -57,7 +57,6 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; -import org.joda.time.DateTime; import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -96,6 +95,7 @@ import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Iterables.transform; import static java.lang.String.format; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; @@ -506,12 +506,12 @@ public TaskInfo abortTask(TaskId taskId) public void removeOldTasks() { - DateTime oldestAllowedTask = DateTime.now().minus(infoCacheTime.toMillis()); + long oldestAllowedTaskInMillis = currentTimeMillis() - infoCacheTime.toMillis(); for (TaskInfo taskInfo : filter(transform(tasks.asMap().values(), SqlTask::getTaskInfo), notNull())) { TaskId taskId = taskInfo.getTaskId(); try { - DateTime endTime = taskInfo.getStats().getEndTime(); - if (endTime != null && endTime.isBefore(oldestAllowedTask)) { + long endTimeInMillis = taskInfo.getStats().getEndTimeInMillis(); + if (endTimeInMillis != 0 && endTimeInMillis < oldestAllowedTaskInMillis) { tasks.asMap().remove(taskId); } } @@ -523,8 +523,8 @@ public void removeOldTasks() public void failAbandonedTasks() { - DateTime now = DateTime.now(); - DateTime oldestAllowedHeartbeat = now.minus(clientTimeout.toMillis()); + long nowInMills = currentTimeMillis(); + long oldestAllowedHeartbeatInMillis = nowInMills - clientTimeout.toMillis(); for (SqlTask sqlTask : tasks.asMap().values()) { try { TaskInfo taskInfo = sqlTask.getTaskInfo(); @@ -532,10 +532,10 @@ public void failAbandonedTasks() if (taskStatus.getState().isDone()) { continue; } - DateTime lastHeartbeat = taskInfo.getLastHeartbeat(); - if (lastHeartbeat != null && lastHeartbeat.isBefore(oldestAllowedHeartbeat)) { + long lastHeartbeatInMillis = taskInfo.getLastHeartbeatInMillis(); + if (lastHeartbeatInMillis != 0 && lastHeartbeatInMillis < oldestAllowedHeartbeatInMillis) { log.info("Failing abandoned task %s", taskInfo.getTaskId()); - sqlTask.failed(new PrestoException(ABANDONED_TASK, format("Task %s has not been accessed since %s: currentTime %s", taskInfo.getTaskId(), lastHeartbeat, now))); + sqlTask.failed(new PrestoException(ABANDONED_TASK, format("Task %s has not been accessed since %sms: currentTime %sms", taskInfo.getTaskId(), lastHeartbeatInMillis, nowInMills))); } } catch (RuntimeException e) { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java index bad065f2867a7..f03c5e364bc86 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import org.joda.time.DateTime; import java.util.ArrayList; import java.util.HashMap; @@ -63,7 +62,7 @@ public static StageExecutionInfo create( StageExecutionState state, Optional failureInfo, List taskInfos, - DateTime schedulingComplete, + long schedulingCompleteInMillis, DistributionSnapshot getSplitDistribution, RuntimeStats stageRuntimeStats, long peakUserMemoryReservation, @@ -98,7 +97,7 @@ public static StageExecutionInfo create( } StageExecutionStats stageExecutionStats = new StageExecutionStats( - schedulingComplete, + schedulingCompleteInMillis, getSplitDistribution, taskStatsAggregator.totalTaskCount, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java index 8129747c5bdd5..88905a33c45af 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java @@ -23,7 +23,6 @@ import com.facebook.presto.operator.TaskStats; import com.facebook.presto.util.Failures; import com.google.common.collect.ImmutableList; -import org.joda.time.DateTime; import javax.annotation.concurrent.ThreadSafe; @@ -66,6 +65,7 @@ import static io.airlift.units.Duration.succinctNanos; import static java.lang.Math.max; import static java.lang.Math.min; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -83,7 +83,7 @@ public class StageExecutionStateMachine private final StateMachine> finalInfo; private final AtomicReference failureCause = new AtomicReference<>(); - private final AtomicReference schedulingComplete = new AtomicReference<>(); + private final AtomicLong schedulingComplete = new AtomicLong(); private final Distribution getSplitDistribution = new Distribution(); private final AtomicLong peakUserMemory = new AtomicLong(); @@ -146,7 +146,7 @@ public synchronized boolean transitionToSchedulingSplits() public synchronized boolean transitionToScheduled() { - schedulingComplete.compareAndSet(null, DateTime.now()); + schedulingComplete.compareAndSet(0, currentTimeMillis()); return state.setIf(SCHEDULED, currentState -> currentState == PLANNED || currentState == SCHEDULING || currentState == FINISHED_TASK_SCHEDULING || currentState == SCHEDULING_SPLITS); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java index 82e944f896c30..c7868868c6883 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.concurrent.Immutable; @@ -41,7 +40,7 @@ @Immutable public class StageExecutionStats { - private final DateTime schedulingComplete; + private final long schedulingCompleteInMillis; private final DistributionSnapshot getSplitDistribution; @@ -95,7 +94,7 @@ public class StageExecutionStats @JsonCreator public StageExecutionStats( - @JsonProperty("schedulingComplete") DateTime schedulingComplete, + @JsonProperty("schedulingCompleteInMillis") long schedulingCompleteInMillis, @JsonProperty("getSplitDistribution") DistributionSnapshot getSplitDistribution, @@ -145,7 +144,7 @@ public StageExecutionStats( @JsonProperty("operatorSummaries") List operatorSummaries, @JsonProperty("runtimeStats") RuntimeStats runtimeStats) { - this.schedulingComplete = schedulingComplete; + this.schedulingCompleteInMillis = schedulingCompleteInMillis; this.getSplitDistribution = requireNonNull(getSplitDistribution, "getSplitDistribution is null"); checkArgument(totalTasks >= 0, "totalTasks is negative"); @@ -219,9 +218,9 @@ public StageExecutionStats( } @JsonProperty - public DateTime getSchedulingComplete() + public long getSchedulingCompleteInMillis() { - return schedulingComplete; + return schedulingCompleteInMillis; } @JsonProperty @@ -466,7 +465,7 @@ public BasicStageExecutionStats toBasicStageStats(StageExecutionState stageExecu public static StageExecutionStats zero(int stageId) { return new StageExecutionStats( - null, + 0L, new Distribution().snapshot(), 0, 0, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java index ee9889b8e283d..a714f87250179 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java @@ -35,7 +35,10 @@ import static com.facebook.presto.execution.TaskStatus.initialTaskStatus; import static com.facebook.presto.execution.buffer.BufferState.OPEN; import static com.facebook.presto.metadata.MetadataUpdates.DEFAULT_METADATA_UPDATES; +import static com.facebook.presto.util.DateTimeUtils.toTimeStampInMillis; import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; @Immutable @@ -44,7 +47,7 @@ public class TaskInfo { private final TaskId taskId; private final TaskStatus taskStatus; - private final DateTime lastHeartbeat; + private final long lastHeartbeatInMillis; private final OutputBufferInfo outputBuffers; private final Set noMoreSplits; private final TaskStats stats; @@ -53,6 +56,29 @@ public class TaskInfo private final MetadataUpdates metadataUpdates; private final String nodeId; + public TaskInfo(TaskId taskId, + TaskStatus taskStatus, + long lastHeartbeatInMillis, + OutputBufferInfo outputBuffers, + Set noMoreSplits, + TaskStats stats, + boolean needsPlan, + MetadataUpdates metadataUpdates, + String nodeId) + { + this.taskId = requireNonNull(taskId, "taskId is null"); + this.taskStatus = requireNonNull(taskStatus, "taskStatus is null"); + checkArgument(lastHeartbeatInMillis >= 0, "lastHeartbeat is negative"); + this.lastHeartbeatInMillis = lastHeartbeatInMillis; + this.outputBuffers = requireNonNull(outputBuffers, "outputBuffers is null"); + this.noMoreSplits = requireNonNull(noMoreSplits, "noMoreSplits is null"); + this.stats = requireNonNull(stats, "stats is null"); + + this.needsPlan = needsPlan; + this.metadataUpdates = metadataUpdates; + this.nodeId = requireNonNull(nodeId, "nodeId is null"); + } + @JsonCreator @ThriftConstructor public TaskInfo( @@ -66,16 +92,15 @@ public TaskInfo( @JsonProperty("metadataUpdates") MetadataUpdates metadataUpdates, @JsonProperty("nodeId") String nodeId) { - this.taskId = requireNonNull(taskId, "taskId is null"); - this.taskStatus = requireNonNull(taskStatus, "taskStatus is null"); - this.lastHeartbeat = requireNonNull(lastHeartbeat, "lastHeartbeat is null"); - this.outputBuffers = requireNonNull(outputBuffers, "outputBuffers is null"); - this.noMoreSplits = requireNonNull(noMoreSplits, "noMoreSplits is null"); - this.stats = requireNonNull(stats, "stats is null"); - - this.needsPlan = needsPlan; - this.metadataUpdates = metadataUpdates; - this.nodeId = requireNonNull(nodeId, "nodeId is null"); + this(taskId, + taskStatus, + toTimeStampInMillis(lastHeartbeat), + outputBuffers, + noMoreSplits, + stats, + needsPlan, + metadataUpdates, + nodeId); } @JsonProperty @@ -96,7 +121,12 @@ public TaskStatus getTaskStatus() @ThriftField(3) public DateTime getLastHeartbeat() { - return lastHeartbeat; + return new DateTime(lastHeartbeatInMillis); + } + + public long getLastHeartbeatInMillis() + { + return lastHeartbeatInMillis; } @JsonProperty @@ -147,7 +177,7 @@ public TaskInfo summarize() return new TaskInfo( taskId, taskStatus, - lastHeartbeat, + lastHeartbeatInMillis, outputBuffers.summarize(), noMoreSplits, stats.summarizeFinal(), @@ -158,7 +188,7 @@ public TaskInfo summarize() return new TaskInfo( taskId, taskStatus, - lastHeartbeat, + lastHeartbeatInMillis, outputBuffers.summarize(), noMoreSplits, stats.summarize(), @@ -181,7 +211,7 @@ public static TaskInfo createInitialTask(TaskId taskId, URI location, List taskState; @@ -54,9 +53,9 @@ public void stateChanged(TaskState newState) }); } - public DateTime getCreatedTime() + public long getCreatedTimeInMillis() { - return createdTime; + return createdTimeInMillis; } public TaskId getTaskId() diff --git a/presto-main/src/main/java/com/facebook/presto/failureDetector/HeartbeatFailureDetector.java b/presto-main/src/main/java/com/facebook/presto/failureDetector/HeartbeatFailureDetector.java index 11b3320e971ba..75be9961bf310 100644 --- a/presto-main/src/main/java/com/facebook/presto/failureDetector/HeartbeatFailureDetector.java +++ b/presto-main/src/main/java/com/facebook/presto/failureDetector/HeartbeatFailureDetector.java @@ -34,7 +34,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import io.airlift.units.Duration; -import org.joda.time.DateTime; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -60,6 +59,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; @@ -411,8 +411,8 @@ public static class Stats private final DecayCounter recentRequests; private final DecayCounter recentFailures; private final DecayCounter recentSuccesses; - private final AtomicReference lastRequestTime = new AtomicReference<>(); - private final AtomicReference lastResponseTime = new AtomicReference<>(); + private final AtomicLong lastRequestTimeInMillis = new AtomicLong(); + private final AtomicLong lastResponseTimeInMillis = new AtomicLong(); private final AtomicReference lastFailureException = new AtomicReference<>(); @GuardedBy("this") @@ -429,19 +429,19 @@ public Stats(URI uri, int exponentialDecaySeconds) public void recordStart() { recentRequests.add(1); - lastRequestTime.set(new DateTime()); + lastRequestTimeInMillis.set(System.currentTimeMillis()); } public void recordSuccess() { recentSuccesses.add(1); - lastResponseTime.set(new DateTime()); + lastResponseTimeInMillis.set(System.currentTimeMillis()); } public void recordFailure(Exception exception) { recentFailures.add(1); - lastResponseTime.set(new DateTime()); + lastResponseTimeInMillis.set(System.currentTimeMillis()); lastFailureException.set(exception); Throwable cause = exception; @@ -496,15 +496,15 @@ public double getRecentFailureRatio() } @JsonProperty - public DateTime getLastRequestTime() + public long getLastRequestTimeInMillis() { - return lastRequestTime.get(); + return lastRequestTimeInMillis.get(); } @JsonProperty - public DateTime getLastResponseTime() + public long getLastResponseTimeInMillis() { - return lastResponseTime.get(); + return lastResponseTimeInMillis.get(); } @JsonIgnore diff --git a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryLeakDetector.java b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryLeakDetector.java index 10be61ba214b1..74b0b2e3c71aa 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryLeakDetector.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryLeakDetector.java @@ -18,7 +18,6 @@ import com.facebook.presto.spi.QueryId; import com.google.common.collect.ImmutableSet; import io.airlift.units.DataSize; -import org.joda.time.DateTime; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -30,8 +29,7 @@ import static com.facebook.presto.execution.QueryState.RUNNING; import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static org.joda.time.DateTime.now; -import static org.joda.time.Seconds.secondsBetween; +import static java.lang.System.currentTimeMillis; @ThreadSafe public class ClusterMemoryLeakDetector @@ -40,7 +38,7 @@ public class ClusterMemoryLeakDetector // It may take some time to remove a query's memory reservations from the worker nodes, that's why // we check to see whether some time has passed after the query finishes to claim that it is leaked. - private static final int DEFAULT_LEAK_CLAIM_DELTA_SEC = 60; + private static final int DEFAULT_LEAK_CLAIM_DELTA_MILLIS = 60_000; @GuardedBy("this") private Set leakedQueries; @@ -63,7 +61,7 @@ void checkForMemoryLeaks(Map> queryIdToInfo, M long leakedBytesThisTime = leakedQueryReservations.values().stream().reduce(0L, Long::sum); if (!leakedQueryReservations.isEmpty()) { log.warn("Memory leak of %s detected. The following queries are already finished, " + - "but they have memory reservations on some worker node(s): %s", + "but they have memory reservations on some worker node(s): %s", DataSize.succinctBytes(leakedBytes), leakedQueryReservations); } @@ -82,9 +80,9 @@ private static boolean isLeaked(Map> queryIdTo return true; } - Optional queryEndTime = queryInfo.flatMap(qi -> Optional.ofNullable(qi.getState() == RUNNING ? null : qi.getQueryStats().getEndTime())); + Optional queryEndTimeInMillis = queryInfo.flatMap(qi -> Optional.ofNullable(qi.getState() == RUNNING ? null : qi.getQueryStats().getEndTimeInMillis())); - return queryEndTime.map(ts -> secondsBetween(ts, now()).getSeconds() >= DEFAULT_LEAK_CLAIM_DELTA_SEC).orElse(false); + return queryEndTimeInMillis.map(ts -> (currentTimeMillis() - ts) >= DEFAULT_LEAK_CLAIM_DELTA_MILLIS).orElse(false); } synchronized boolean wasQueryPossiblyLeaked(QueryId queryId) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/DriverContext.java b/presto-main/src/main/java/com/facebook/presto/operator/DriverContext.java index ef8a51ede0727..95bac0519e9ef 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/DriverContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/DriverContext.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.Duration; -import org.joda.time.DateTime; import java.util.List; import java.util.Optional; @@ -44,6 +43,8 @@ import static com.google.common.collect.Iterables.transform; import static io.airlift.units.Duration.succinctNanos; import static java.lang.Math.max; +import static java.lang.System.currentTimeMillis; +import static java.lang.System.nanoTime; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -60,8 +61,8 @@ public class DriverContext private final AtomicBoolean finished = new AtomicBoolean(); - private final DateTime createdTime = DateTime.now(); - private final long createNanos = System.nanoTime(); + private final long createdTimeInMillis = currentTimeMillis(); + private final long createNanos = nanoTime(); private final AtomicLong startNanos = new AtomicLong(); private final AtomicLong endNanos = new AtomicLong(); @@ -71,8 +72,8 @@ public class DriverContext private final AtomicReference blockedMonitor = new AtomicReference<>(); private final AtomicLong blockedWallNanos = new AtomicLong(); - private final AtomicReference executionStartTime = new AtomicReference<>(); - private final AtomicReference executionEndTime = new AtomicReference<>(); + private final AtomicLong executionStartTime = new AtomicLong(); + private final AtomicLong executionEndTime = new AtomicLong(); private final MemoryTrackingContext driverMemoryContext; @@ -150,8 +151,8 @@ public Session getSession() public void startProcessTimer() { // Must update startNanos first so that the value is valid once executionStartTime is not null - if (executionStartTime.get() == null && startNanos.compareAndSet(0, System.nanoTime())) { - executionStartTime.set(DateTime.now()); + if (executionStartTime.get() == 0 && startNanos.compareAndSet(0, nanoTime())) { + executionStartTime.set(currentTimeMillis()); pipelineContext.start(); } } @@ -182,8 +183,8 @@ public void finished() return; } // Must update endNanos first, so that the value is valid after executionEndTime is not null - endNanos.set(System.nanoTime()); - executionEndTime.set(DateTime.now()); + endNanos.set(nanoTime()); + executionEndTime.set(currentTimeMillis()); pipelineContext.driverFinished(this); } @@ -311,7 +312,7 @@ public long getPhysicalWrittenDataSize() public boolean isExecutionStarted() { - return executionStartTime.get() != null; + return executionStartTime.get() != 0; } public boolean isFullyBlocked() @@ -332,12 +333,12 @@ public DriverStats getDriverStats() } // startNanos is always valid once executionStartTime is not null - DateTime executionStartTime = this.executionStartTime.get(); - Duration queuedTime = new Duration(nanosBetween(createNanos, executionStartTime == null ? System.nanoTime() : startNanos.get()), NANOSECONDS); + long executionStartTimeInMillis = this.executionStartTime.get(); + Duration queuedTime = new Duration(nanosBetween(createNanos, executionStartTimeInMillis == 0 ? nanoTime() : startNanos.get()), NANOSECONDS); // endNanos is always valid once executionStartTime is not null - DateTime executionEndTime = this.executionEndTime.get(); - Duration elapsedTime = new Duration(nanosBetween(createNanos, executionEndTime == null ? System.nanoTime() : endNanos.get()), NANOSECONDS); + long executionEndTimeInMillis = this.executionEndTime.get(); + Duration elapsedTime = new Duration(nanosBetween(createNanos, executionEndTimeInMillis == 0 ? nanoTime() : endNanos.get()), NANOSECONDS); List operators = ImmutableList.copyOf(transform(operatorContexts, OperatorContext::getOperatorStats)); OperatorStats inputOperator = getFirst(operators, null); @@ -384,9 +385,9 @@ public DriverStats getDriverStats() return new DriverStats( lifespan, - createdTime, - executionStartTime, - executionEndTime, + createdTimeInMillis, + executionStartTimeInMillis, + executionEndTimeInMillis, queuedTime.convertToMostSuccinctTimeUnit(), elapsedTime.convertToMostSuccinctTimeUnit(), driverMemoryContext.getUserMemory(), @@ -444,7 +445,7 @@ private static long nanosBetween(long start, long end) private class BlockedMonitor implements Runnable { - private final long start = System.nanoTime(); + private final long start = nanoTime(); private boolean finished; @Override @@ -462,7 +463,7 @@ public void run() public long getBlockedTime() { - return nanosBetween(start, System.nanoTime()); + return nanosBetween(start, nanoTime()); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/DriverStats.java b/presto-main/src/main/java/com/facebook/presto/operator/DriverStats.java index 3e248ddf2d95a..7712e3a49fa84 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/DriverStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/DriverStats.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; @@ -31,6 +30,7 @@ import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -40,9 +40,9 @@ public class DriverStats { private final Lifespan lifespan; - private final DateTime createTime; - private final DateTime startTime; - private final DateTime endTime; + private final long createTimeInMillis; + private final long startTimeInMillis; + private final long endTimeInMillis; private final Duration queuedTime; private final Duration elapsedTime; @@ -77,9 +77,9 @@ public DriverStats() { this.lifespan = null; - this.createTime = DateTime.now(); - this.startTime = null; - this.endTime = null; + this.createTimeInMillis = currentTimeMillis(); + this.startTimeInMillis = 0L; + this.endTimeInMillis = 0L; this.queuedTime = new Duration(0, MILLISECONDS); this.elapsedTime = new Duration(0, MILLISECONDS); @@ -115,9 +115,9 @@ public DriverStats() public DriverStats( @JsonProperty("lifespan") Lifespan lifespan, - @JsonProperty("createTime") DateTime createTime, - @JsonProperty("startTime") DateTime startTime, - @JsonProperty("endTime") DateTime endTime, + @JsonProperty("createTimeInMillis") long createTimeInMillis, + @JsonProperty("startTimeInMillis") long startTimeInMillis, + @JsonProperty("endTimeInMillis") long endTimeInMillis, @JsonProperty("queuedTime") Duration queuedTime, @JsonProperty("elapsedTime") Duration elapsedTime, @@ -149,9 +149,10 @@ public DriverStats( { this.lifespan = lifespan; - this.createTime = requireNonNull(createTime, "createTime is null"); - this.startTime = startTime; - this.endTime = endTime; + checkArgument(createTimeInMillis >= 0, "createTimeInMillis is negative"); + this.createTimeInMillis = createTimeInMillis; + this.startTimeInMillis = startTimeInMillis; + this.endTimeInMillis = endTimeInMillis; this.queuedTime = requireNonNull(queuedTime, "queuedTime is null"); this.elapsedTime = requireNonNull(elapsedTime, "elapsedTime is null"); @@ -205,25 +206,25 @@ public Lifespan getLifespan() @JsonProperty @ThriftField(2) - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return createTime; + return createTimeInMillis; } @Nullable @JsonProperty @ThriftField(3) - public DateTime getStartTime() + public long getStartTimeInMillis() { - return startTime; + return startTimeInMillis; } @Nullable @JsonProperty @ThriftField(4) - public DateTime getEndTime() + public long getEndTimeInMillis() { - return endTime; + return endTimeInMillis; } @JsonProperty diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PageBufferClient.java b/presto-main/src/main/java/com/facebook/presto/operator/PageBufferClient.java index 0d7a39fdb2dab..c04addfcd2ba9 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PageBufferClient.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PageBufferClient.java @@ -26,7 +26,6 @@ import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.DataSize; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; @@ -54,6 +53,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.isNullOrEmpty; import static java.lang.String.format; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -95,7 +95,7 @@ public interface ClientCallback @GuardedBy("this") private ListenableFuture future; @GuardedBy("this") - private DateTime lastUpdate = DateTime.now(); + private long lastUpdate = currentTimeMillis(); @GuardedBy("this") private long token; @GuardedBy("this") @@ -205,7 +205,7 @@ public void close() this.future = null; - lastUpdate = DateTime.now(); + lastUpdate = currentTimeMillis(); } if (future != null && !future.isDone()) { @@ -240,7 +240,7 @@ public synchronized void scheduleRequest(DataSize maxResponseSize) } }, delayNanos, NANOSECONDS); - lastUpdate = DateTime.now(); + lastUpdate = currentTimeMillis(); requestsScheduled.incrementAndGet(); } @@ -258,7 +258,7 @@ private synchronized void initiateRequest(DataSize maxResponseSize) sendGetResults(maxResponseSize); } - lastUpdate = DateTime.now(); + lastUpdate = currentTimeMillis(); } private synchronized void sendGetResults(DataSize maxResponseSize) @@ -349,7 +349,7 @@ public void onSuccess(PagesResponse result) if (future == resultFuture) { future = null; } - lastUpdate = DateTime.now(); + lastUpdate = currentTimeMillis(); } clientCallback.requestComplete(PageBufferClient.this); } @@ -391,7 +391,7 @@ public void onSuccess(@Nullable Object result) if (future == resultFuture) { future = null; } - lastUpdate = DateTime.now(); + lastUpdate = currentTimeMillis(); } requestsCompleted.incrementAndGet(); clientCallback.clientFinished(PageBufferClient.this); @@ -437,7 +437,7 @@ private void handleFailure(Throwable t, ListenableFuture expectedFuture) if (future == expectedFuture) { future = null; } - lastUpdate = DateTime.now(); + lastUpdate = currentTimeMillis(); } clientCallback.requestComplete(PageBufferClient.this); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PageBufferClientStatus.java b/presto-main/src/main/java/com/facebook/presto/operator/PageBufferClientStatus.java index a1a7c28e29476..9f7f3eeca9017 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PageBufferClientStatus.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PageBufferClientStatus.java @@ -24,6 +24,7 @@ import java.util.OptionalInt; import java.util.OptionalLong; +import static com.facebook.presto.util.DateTimeUtils.toTimeStampInMillis; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; @@ -32,7 +33,7 @@ public class PageBufferClientStatus { private final URI uri; private final String state; - private final DateTime lastUpdate; + private final long lastUpdateInMillis; private final long rowsReceived; private final int pagesReceived; // use optional to keep the output size down, since this renders for every destination @@ -43,6 +44,32 @@ public class PageBufferClientStatus private final int requestsFailed; private final String httpRequestState; + public PageBufferClientStatus( + URI uri, + String state, + long lastUpdateInMillis, + long rowsReceived, + int pagesReceived, + OptionalLong rowsRejected, + OptionalInt pagesRejected, + int requestsScheduled, + int requestsCompleted, + int requestsFailed, + String httpRequestState) + { + this.uri = uri; + this.state = state; + this.lastUpdateInMillis = lastUpdateInMillis; + this.rowsReceived = rowsReceived; + this.pagesReceived = pagesReceived; + this.rowsRejected = requireNonNull(rowsRejected, "rowsRejected is null"); + this.pagesRejected = requireNonNull(pagesRejected, "pagesRejected is null"); + this.requestsScheduled = requestsScheduled; + this.requestsCompleted = requestsCompleted; + this.requestsFailed = requestsFailed; + this.httpRequestState = httpRequestState; + } + @JsonCreator @ThriftConstructor public PageBufferClientStatus(@JsonProperty("uri") URI uri, @@ -57,17 +84,17 @@ public PageBufferClientStatus(@JsonProperty("uri") URI uri, @JsonProperty("requestsFailed") int requestsFailed, @JsonProperty("httpRequestState") String httpRequestState) { - this.uri = uri; - this.state = state; - this.lastUpdate = lastUpdate; - this.rowsReceived = rowsReceived; - this.pagesReceived = pagesReceived; - this.rowsRejected = requireNonNull(rowsRejected, "rowsRejected is null"); - this.pagesRejected = requireNonNull(pagesRejected, "pagesRejected is null"); - this.requestsScheduled = requestsScheduled; - this.requestsCompleted = requestsCompleted; - this.requestsFailed = requestsFailed; - this.httpRequestState = httpRequestState; + this(uri, + state, + toTimeStampInMillis(lastUpdate), + rowsReceived, + pagesReceived, + rowsRejected, + pagesRejected, + requestsScheduled, + requestsCompleted, + requestsFailed, + httpRequestState); } @JsonProperty @@ -88,7 +115,12 @@ public String getState() @ThriftField(3) public DateTime getLastUpdate() { - return lastUpdate; + return new DateTime(lastUpdateInMillis); + } + + public long getLastUpdateInMillis() + { + return lastUpdateInMillis; } @JsonProperty @@ -153,7 +185,7 @@ public String toString() return toStringHelper(this) .add("uri", uri) .add("state", state) - .add("lastUpdate", lastUpdate) + .add("lastUpdateInMillis", lastUpdateInMillis) .add("rowsReceived", rowsReceived) .add("pagesReceived", pagesReceived) .add("httpRequestState", httpRequestState) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PipelineContext.java b/presto-main/src/main/java/com/facebook/presto/operator/PipelineContext.java index fd93b7f722069..99851036dc02d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PipelineContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PipelineContext.java @@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; -import org.joda.time.DateTime; import javax.annotation.concurrent.ThreadSafe; @@ -43,10 +42,10 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.stream.Collectors.toList; @@ -71,9 +70,9 @@ public class PipelineContext private final AtomicInteger completedDrivers = new AtomicInteger(); private final AtomicLong completedSplitsWeight = new AtomicLong(); - private final AtomicReference executionStartTime = new AtomicReference<>(); - private final AtomicReference lastExecutionStartTime = new AtomicReference<>(); - private final AtomicReference lastExecutionEndTime = new AtomicReference<>(); + private final AtomicLong executionStartTime = new AtomicLong(); + private final AtomicLong lastExecutionStartTime = new AtomicLong(); + private final AtomicLong lastExecutionEndTime = new AtomicLong(); private final Distribution queuedTime = new Distribution(); private final Distribution elapsedTime = new Distribution(); @@ -182,7 +181,7 @@ public void driverFinished(DriverContext driverContext) } // always update last execution end time - lastExecutionEndTime.set(DateTime.now()); + lastExecutionEndTime.set(currentTimeMillis()); DriverStats driverStats = driverContext.getDriverStats(); @@ -221,8 +220,8 @@ public void driverFinished(DriverContext driverContext) public void start() { - DateTime now = DateTime.now(); - executionStartTime.compareAndSet(null, now); + long now = currentTimeMillis(); + executionStartTime.compareAndSet(0, now); // always update last execution start time lastExecutionStartTime.set(now); @@ -344,10 +343,10 @@ public PipelineStats getPipelineStats() { // check for end state to avoid callback ordering problems if (taskContext.getState().isDone()) { - DateTime now = DateTime.now(); - executionStartTime.compareAndSet(null, now); - lastExecutionStartTime.compareAndSet(null, now); - lastExecutionEndTime.compareAndSet(null, now); + long now = currentTimeMillis(); + executionStartTime.compareAndSet(0, now); + lastExecutionStartTime.compareAndSet(0, now); + lastExecutionEndTime.compareAndSet(0, now); } int completedDrivers = this.completedDrivers.get(); @@ -389,7 +388,7 @@ public PipelineStats getPipelineStats() DriverStats driverStats = driverContext.getDriverStats(); drivers.add(driverStats); pipelineStatusBuilder.accumulate(driverStats, driverContext.getSplitWeight()); - if (driverStats.getStartTime() != null && driverStats.getEndTime() == null) { + if (driverStats.getStartTimeInMillis() != 0 && driverStats.getEndTimeInMillis() == 0) { // driver has started running, but not yet completed hasUnfinishedDrivers = true; unfinishedDriversFullyBlocked &= driverStats.isFullyBlocked(); @@ -556,7 +555,7 @@ else if (driverContext.isFullyBlocked()) { public void accumulate(DriverStats driverStats, long splitWeight) { - if (driverStats.getStartTime() == null) { + if (driverStats.getStartTimeInMillis() == 0) { // driver has not started running physicallyQueuedDrivers++; } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PipelineStats.java b/presto-main/src/main/java/com/facebook/presto/operator/PipelineStats.java index 4835609d2b444..ad5f7212116f6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PipelineStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PipelineStats.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.joda.time.DateTime; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; @@ -38,9 +37,9 @@ public class PipelineStats { private final int pipelineId; - private final DateTime firstStartTime; - private final DateTime lastStartTime; - private final DateTime lastEndTime; + private final long firstStartTimeInMillis; + private final long lastStartTimeInMillis; + private final long lastEndTimeInMillis; private final boolean inputPipeline; private final boolean outputPipeline; @@ -89,9 +88,9 @@ public class PipelineStats public PipelineStats( @JsonProperty("pipelineId") int pipelineId, - @JsonProperty("firstStartTime") DateTime firstStartTime, - @JsonProperty("lastStartTime") DateTime lastStartTime, - @JsonProperty("lastEndTime") DateTime lastEndTime, + @JsonProperty("firstStartTimeInMillis") long firstStartTimeInMillis, + @JsonProperty("lastStartTimeInMillis") long lastStartTimeInMillis, + @JsonProperty("lastEndTimeInMillis") long lastEndTimeInMillis, @JsonProperty("inputPipeline") boolean inputPipeline, @JsonProperty("outputPipeline") boolean outputPipeline, @@ -137,9 +136,9 @@ public PipelineStats( { this.pipelineId = pipelineId; - this.firstStartTime = firstStartTime; - this.lastStartTime = lastStartTime; - this.lastEndTime = lastEndTime; + this.firstStartTimeInMillis = firstStartTimeInMillis; + this.lastStartTimeInMillis = lastStartTimeInMillis; + this.lastEndTimeInMillis = lastEndTimeInMillis; this.inputPipeline = inputPipeline; this.outputPipeline = outputPipeline; @@ -206,25 +205,25 @@ public int getPipelineId() @Nullable @JsonProperty @ThriftField(2) - public DateTime getFirstStartTime() + public long getFirstStartTimeInMillis() { - return firstStartTime; + return firstStartTimeInMillis; } @Nullable @JsonProperty @ThriftField(3) - public DateTime getLastStartTime() + public long getLastStartTimeInMillis() { - return lastStartTime; + return lastStartTimeInMillis; } @Nullable @JsonProperty @ThriftField(4) - public DateTime getLastEndTime() + public long getLastEndTimeInMillis() { - return lastEndTime; + return lastEndTimeInMillis; } @JsonProperty @@ -448,9 +447,9 @@ public PipelineStats summarize() { return new PipelineStats( pipelineId, - firstStartTime, - lastStartTime, - lastEndTime, + firstStartTimeInMillis, + lastStartTimeInMillis, + lastEndTimeInMillis, inputPipeline, outputPipeline, totalDrivers, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java b/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java index e215dbfd23f68..b75c5ead5bede 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java @@ -41,7 +41,6 @@ import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.DataSize; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -55,7 +54,6 @@ import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; import static com.google.common.base.Preconditions.checkArgument; @@ -68,6 +66,8 @@ import static io.airlift.units.DataSize.succinctBytes; import static java.lang.Math.max; import static java.lang.Math.toIntExact; +import static java.lang.System.currentTimeMillis; +import static java.lang.System.nanoTime; import static java.util.Comparator.comparing; import static java.util.Comparator.reverseOrder; import static java.util.Objects.requireNonNull; @@ -85,7 +85,7 @@ public class TaskContext private final ScheduledExecutorService yieldExecutor; private final Session session; - private final long createNanos = System.nanoTime(); + private final long createNanos = nanoTime(); private final AtomicLong startNanos = new AtomicLong(); private final AtomicLong startFullGcCount = new AtomicLong(-1); @@ -94,9 +94,9 @@ public class TaskContext private final AtomicLong endFullGcCount = new AtomicLong(-1); private final AtomicLong endFullGcTimeNanos = new AtomicLong(-1); - private final AtomicReference executionStartTime = new AtomicReference<>(); - private final AtomicReference lastExecutionStartTime = new AtomicReference<>(); - private final AtomicReference executionEndTime = new AtomicReference<>(); + private final AtomicLong executionStartTime = new AtomicLong(); + private final AtomicLong lastExecutionStartTime = new AtomicLong(); + private final AtomicLong executionEndTime = new AtomicLong(); private final Set completedDriverGroups = newConcurrentHashSet(); @@ -237,9 +237,9 @@ public Session getSession() public void start() { - DateTime now = DateTime.now(); - executionStartTime.compareAndSet(null, now); - startNanos.compareAndSet(0, System.nanoTime()); + long now = currentTimeMillis(); + executionStartTime.compareAndSet(0, now); + startNanos.compareAndSet(0, nanoTime()); startFullGcCount.compareAndSet(-1, gcMonitor.getMajorGcCount()); startFullGcTimeNanos.compareAndSet(-1, gcMonitor.getMajorGcTime().roundTo(NANOSECONDS)); @@ -250,23 +250,23 @@ public void start() private void updateStatsIfDone(TaskState newState) { if (newState.isDone()) { - DateTime now = DateTime.now(); + long now = currentTimeMillis(); long majorGcCount = gcMonitor.getMajorGcCount(); long majorGcTime = gcMonitor.getMajorGcTime().roundTo(NANOSECONDS); // before setting the end times, make sure a start has been recorded - executionStartTime.compareAndSet(null, now); - startNanos.compareAndSet(0, System.nanoTime()); + executionStartTime.compareAndSet(0, now); + startNanos.compareAndSet(0, nanoTime()); startFullGcCount.compareAndSet(-1, majorGcCount); startFullGcTimeNanos.compareAndSet(-1, majorGcTime); // Only update last start time, if the nothing was started - lastExecutionStartTime.compareAndSet(null, now); + lastExecutionStartTime.compareAndSet(0, now); // use compare and set from initial value to avoid overwriting if there // were a duplicate notification, which shouldn't happen - executionEndTime.compareAndSet(null, now); - endNanos.compareAndSet(0, System.nanoTime()); + executionEndTime.compareAndSet(0, now); + endNanos.compareAndSet(0, nanoTime()); endFullGcCount.compareAndSet(-1, majorGcCount); endFullGcTimeNanos.compareAndSet(-1, majorGcTime); } @@ -489,8 +489,8 @@ public TaskStats getTaskStats() boolean runningPipelinesFullyBlocked = true; for (PipelineStats pipeline : pipelineStats) { - if (pipeline.getLastEndTime() != null) { - lastExecutionEndTime = max(pipeline.getLastEndTime().getMillis(), lastExecutionEndTime); + if (pipeline.getLastEndTimeInMillis() != 0) { + lastExecutionEndTime = max(pipeline.getLastEndTimeInMillis(), lastExecutionEndTime); } if (pipeline.getRunningDrivers() > 0 || pipeline.getRunningPartitionedDrivers() > 0 || pipeline.getBlockedDrivers() > 0) { // pipeline is running @@ -534,7 +534,7 @@ public TaskStats getTaskStats() long startNanos = this.startNanos.get(); if (startNanos == 0) { - startNanos = System.nanoTime(); + startNanos = nanoTime(); } long queuedTimeInNanos = startNanos - createNanos; @@ -559,13 +559,13 @@ public TaskStats getTaskStats() if (lastTaskStatCallNanos == 0) { lastTaskStatCallNanos = startNanos; } - double sinceLastPeriodMillis = (System.nanoTime() - lastTaskStatCallNanos) / 1_000_000.0; + double sinceLastPeriodMillis = (nanoTime() - lastTaskStatCallNanos) / 1_000_000.0; long averageUserMemoryForLastPeriod = (userMemory + lastUserMemoryReservation) / 2; long averageTotalMemoryForLastPeriod = (userMemory + systemMemory + lastTotalMemoryReservation) / 2; cumulativeUserMemory.addAndGet(averageUserMemoryForLastPeriod * sinceLastPeriodMillis); cumulativeTotalMemory.addAndGet(averageTotalMemoryForLastPeriod * sinceLastPeriodMillis); - lastTaskStatCallNanos = System.nanoTime(); + lastTaskStatCallNanos = nanoTime(); lastUserMemoryReservation = userMemory; lastTotalMemoryReservation = systemMemory + userMemory; } @@ -573,10 +573,10 @@ public TaskStats getTaskStats() boolean fullyBlocked = hasRunningPipelines && runningPipelinesFullyBlocked; return new TaskStats( - taskStateMachine.getCreatedTime(), + taskStateMachine.getCreatedTimeInMillis(), executionStartTime.get(), lastExecutionStartTime.get(), - lastExecutionEndTime == 0 ? null : new DateTime(lastExecutionEndTime), + lastExecutionEndTime, executionEndTime.get(), elapsedTimeInNanos, queuedTimeInNanos, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java b/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java index df0a5e2b6e141..4e275878c4376 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java @@ -28,17 +28,18 @@ import java.util.List; import java.util.Set; +import static com.facebook.presto.util.DateTimeUtils.toTimeStampInMillis; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @ThriftStruct public class TaskStats { - private final DateTime createTime; - private final DateTime firstStartTime; - private final DateTime lastStartTime; - private final DateTime lastEndTime; - private final DateTime endTime; + private final long createTimeInMillis; + private final long firstStartTimeInMillis; + private final long lastStartTimeInMillis; + private final long lastEndTimeInMillis; + private final long endTimeInMillis; private final long elapsedTimeInNanos; private final long queuedTimeInNanos; @@ -90,10 +91,54 @@ public class TaskStats // RuntimeStats aggregated at the task level including the metrics exposed in this task and each operator of this task. private final RuntimeStats runtimeStats; + public TaskStats(long createTimeInMillis, long endTimeInMillis) + { + this(createTimeInMillis, + 0L, + 0L, + 0L, + endTimeInMillis, + 0L, + 0L, + 0, + 0, + 0, + 0L, + 0, + 0, + 0L, + 0, + 0, + 0.0, + 0.0, + 0L, + 0L, + 0L, + 0L, + 0L, + 0L, + 0L, + 0L, + 0L, + false, + ImmutableSet.of(), + 0L, + 0L, + 0, + 0L, + 0, + 0L, + 0, + 0L, + 0, + 0L, + ImmutableList.of(), + new RuntimeStats()); + } + public TaskStats(DateTime createTime, DateTime endTime) { - this( - createTime, + this(createTime, null, null, null, @@ -136,6 +181,132 @@ public TaskStats(DateTime createTime, DateTime endTime) new RuntimeStats()); } + public TaskStats( + long createTimeInMillis, + long firstStartTimeInMillis, + long lastStartTimeInMillis, + long lastEndTimeInMillis, + long endTimeInMillis, + long elapsedTimeInNanos, + long queuedTimeInNanos, + + int totalDrivers, + int queuedDrivers, + int queuedPartitionedDrivers, + long queuedPartitionedSplitsWeight, + int runningDrivers, + int runningPartitionedDrivers, + long runningPartitionedSplitsWeight, + int blockedDrivers, + int completedDrivers, + + double cumulativeUserMemory, + double cumulativeTotalMemory, + long userMemoryReservationInBytes, + long revocableMemoryReservationInBytes, + long systemMemoryReservationInBytes, + + long peakTotalMemoryInBytes, + long peakUserMemoryInBytes, + long peakNodeTotalMemoryInBytes, + + long totalScheduledTimeInNanos, + long totalCpuTimeInNanos, + long totalBlockedTimeInNanos, + boolean fullyBlocked, + Set blockedReasons, + + long totalAllocationInBytes, + + long rawInputDataSizeInBytes, + long rawInputPositions, + + long processedInputDataSizeInBytes, + long processedInputPositions, + + long outputDataSizeInBytes, + long outputPositions, + + long physicalWrittenDataSizeInBytes, + + int fullGcCount, + long fullGcTimeInMillis, + + List pipelines, + RuntimeStats runtimeStats) + { + checkArgument(createTimeInMillis >= 0, "createTimeInMillis is negative"); + this.createTimeInMillis = createTimeInMillis; + this.firstStartTimeInMillis = firstStartTimeInMillis; + this.lastStartTimeInMillis = lastStartTimeInMillis; + this.lastEndTimeInMillis = lastEndTimeInMillis; + this.endTimeInMillis = endTimeInMillis; + + this.elapsedTimeInNanos = elapsedTimeInNanos; + this.queuedTimeInNanos = queuedTimeInNanos; + + checkArgument(totalDrivers >= 0, "totalDrivers is negative"); + this.totalDrivers = totalDrivers; + checkArgument(queuedDrivers >= 0, "queuedDrivers is negative"); + this.queuedDrivers = queuedDrivers; + checkArgument(queuedPartitionedDrivers >= 0, "queuedPartitionedDrivers is negative"); + this.queuedPartitionedDrivers = queuedPartitionedDrivers; + checkArgument(queuedPartitionedSplitsWeight >= 0, "queuedPartitionedSplitsWeight must be non-negative"); + this.queuedPartitionedSplitsWeight = queuedPartitionedSplitsWeight; + + checkArgument(runningDrivers >= 0, "runningDrivers is negative"); + this.runningDrivers = runningDrivers; + checkArgument(runningPartitionedDrivers >= 0, "runningPartitionedDrivers is negative"); + this.runningPartitionedDrivers = runningPartitionedDrivers; + checkArgument(runningPartitionedSplitsWeight >= 0, "runningPartitionedSplitsWeight must be non-negative"); + this.runningPartitionedSplitsWeight = runningPartitionedSplitsWeight; + + checkArgument(blockedDrivers >= 0, "blockedDrivers is negative"); + this.blockedDrivers = blockedDrivers; + + checkArgument(completedDrivers >= 0, "completedDrivers is negative"); + this.completedDrivers = completedDrivers; + + this.cumulativeUserMemory = cumulativeUserMemory; + this.cumulativeTotalMemory = cumulativeTotalMemory; + this.userMemoryReservationInBytes = userMemoryReservationInBytes; + this.revocableMemoryReservationInBytes = revocableMemoryReservationInBytes; + this.systemMemoryReservationInBytes = systemMemoryReservationInBytes; + + this.peakTotalMemoryInBytes = peakTotalMemoryInBytes; + this.peakUserMemoryInBytes = peakUserMemoryInBytes; + this.peakNodeTotalMemoryInBytes = peakNodeTotalMemoryInBytes; + + this.totalScheduledTimeInNanos = totalScheduledTimeInNanos; + this.totalCpuTimeInNanos = totalCpuTimeInNanos; + this.totalBlockedTimeInNanos = totalBlockedTimeInNanos; + this.fullyBlocked = fullyBlocked; + this.blockedReasons = ImmutableSet.copyOf(requireNonNull(blockedReasons, "blockedReasons is null")); + + this.totalAllocationInBytes = totalAllocationInBytes; + + this.rawInputDataSizeInBytes = rawInputDataSizeInBytes; + checkArgument(rawInputPositions >= 0, "rawInputPositions is negative"); + this.rawInputPositions = rawInputPositions; + + this.processedInputDataSizeInBytes = processedInputDataSizeInBytes; + checkArgument(processedInputPositions >= 0, "processedInputPositions is negative"); + this.processedInputPositions = processedInputPositions; + + this.outputDataSizeInBytes = outputDataSizeInBytes; + checkArgument(outputPositions >= 0, "outputPositions is negative"); + this.outputPositions = outputPositions; + + this.physicalWrittenDataSizeInBytes = physicalWrittenDataSizeInBytes; + + checkArgument(fullGcCount >= 0, "fullGcCount is negative"); + this.fullGcCount = fullGcCount; + this.fullGcTimeInMillis = fullGcTimeInMillis; + + this.pipelines = ImmutableList.copyOf(requireNonNull(pipelines, "pipelines is null")); + this.runtimeStats = requireNonNull(runtimeStats, "runtimeStats is null"); + } + @JsonCreator @ThriftConstructor public TaskStats( @@ -192,81 +363,71 @@ public TaskStats( @JsonProperty("pipelines") List pipelines, @JsonProperty("runtimeStats") RuntimeStats runtimeStats) { - this.createTime = requireNonNull(createTime, "createTime is null"); - this.firstStartTime = firstStartTime; - this.lastStartTime = lastStartTime; - this.lastEndTime = lastEndTime; - this.endTime = endTime; - this.elapsedTimeInNanos = elapsedTimeInNanos; - this.queuedTimeInNanos = queuedTimeInNanos; - - checkArgument(totalDrivers >= 0, "totalDrivers is negative"); - this.totalDrivers = totalDrivers; - checkArgument(queuedDrivers >= 0, "queuedDrivers is negative"); - this.queuedDrivers = queuedDrivers; - checkArgument(queuedPartitionedDrivers >= 0, "queuedPartitionedDrivers is negative"); - this.queuedPartitionedDrivers = queuedPartitionedDrivers; - checkArgument(queuedPartitionedSplitsWeight >= 0, "queuedPartitionedSplitsWeight must be non-negative"); - this.queuedPartitionedSplitsWeight = queuedPartitionedSplitsWeight; + this(toTimeStampInMillis(createTime), + toTimeStampInMillis(firstStartTime), + toTimeStampInMillis(lastStartTime), + toTimeStampInMillis(lastEndTime), + toTimeStampInMillis(endTime), - checkArgument(runningDrivers >= 0, "runningDrivers is negative"); - this.runningDrivers = runningDrivers; - checkArgument(runningPartitionedDrivers >= 0, "runningPartitionedDrivers is negative"); - this.runningPartitionedDrivers = runningPartitionedDrivers; - checkArgument(runningPartitionedSplitsWeight >= 0, "runningPartitionedSplitsWeight must be non-negative"); - this.runningPartitionedSplitsWeight = runningPartitionedSplitsWeight; - - checkArgument(blockedDrivers >= 0, "blockedDrivers is negative"); - this.blockedDrivers = blockedDrivers; + elapsedTimeInNanos, + queuedTimeInNanos, - checkArgument(completedDrivers >= 0, "completedDrivers is negative"); - this.completedDrivers = completedDrivers; + totalDrivers, + queuedDrivers, + queuedPartitionedDrivers, + queuedPartitionedSplitsWeight, + runningDrivers, + runningPartitionedDrivers, + runningPartitionedSplitsWeight, + blockedDrivers, + completedDrivers, - this.cumulativeUserMemory = cumulativeUserMemory; - this.cumulativeTotalMemory = cumulativeTotalMemory; - this.userMemoryReservationInBytes = userMemoryReservationInBytes; - this.revocableMemoryReservationInBytes = revocableMemoryReservationInBytes; - this.systemMemoryReservationInBytes = systemMemoryReservationInBytes; + cumulativeUserMemory, + cumulativeTotalMemory, + userMemoryReservationInBytes, + revocableMemoryReservationInBytes, + systemMemoryReservationInBytes, - this.peakTotalMemoryInBytes = peakTotalMemoryInBytes; - this.peakUserMemoryInBytes = peakUserMemoryInBytes; - this.peakNodeTotalMemoryInBytes = peakNodeTotalMemoryInBytes; + peakTotalMemoryInBytes, + peakUserMemoryInBytes, + peakNodeTotalMemoryInBytes, - this.totalScheduledTimeInNanos = totalScheduledTimeInNanos; - this.totalCpuTimeInNanos = totalCpuTimeInNanos; - this.totalBlockedTimeInNanos = totalBlockedTimeInNanos; - this.fullyBlocked = fullyBlocked; - this.blockedReasons = ImmutableSet.copyOf(requireNonNull(blockedReasons, "blockedReasons is null")); + totalScheduledTimeInNanos, + totalCpuTimeInNanos, + totalBlockedTimeInNanos, + fullyBlocked, + blockedReasons, - this.totalAllocationInBytes = totalAllocationInBytes; + totalAllocationInBytes, - this.rawInputDataSizeInBytes = rawInputDataSizeInBytes; - checkArgument(rawInputPositions >= 0, "rawInputPositions is negative"); - this.rawInputPositions = rawInputPositions; + rawInputDataSizeInBytes, + rawInputPositions, - this.processedInputDataSizeInBytes = processedInputDataSizeInBytes; - checkArgument(processedInputPositions >= 0, "processedInputPositions is negative"); - this.processedInputPositions = processedInputPositions; + processedInputDataSizeInBytes, + processedInputPositions, - this.outputDataSizeInBytes = outputDataSizeInBytes; - checkArgument(outputPositions >= 0, "outputPositions is negative"); - this.outputPositions = outputPositions; + outputDataSizeInBytes, + outputPositions, - this.physicalWrittenDataSizeInBytes = physicalWrittenDataSizeInBytes; + physicalWrittenDataSizeInBytes, - checkArgument(fullGcCount >= 0, "fullGcCount is negative"); - this.fullGcCount = fullGcCount; - this.fullGcTimeInMillis = fullGcTimeInMillis; + fullGcCount, + fullGcTimeInMillis, - this.pipelines = ImmutableList.copyOf(requireNonNull(pipelines, "pipelines is null")); - this.runtimeStats = requireNonNull(runtimeStats, "runtimeStats is null"); + pipelines, + runtimeStats); } @JsonProperty @ThriftField(1) public DateTime getCreateTime() { - return createTime; + return new DateTime(createTimeInMillis); + } + + public long getCreateTimeInMillis() + { + return createTimeInMillis; } @Nullable @@ -274,7 +435,12 @@ public DateTime getCreateTime() @ThriftField(2) public DateTime getFirstStartTime() { - return firstStartTime; + return new DateTime(firstStartTimeInMillis); + } + + public long getFirstStartTimeInMillis() + { + return firstStartTimeInMillis; } @Nullable @@ -282,7 +448,12 @@ public DateTime getFirstStartTime() @ThriftField(3) public DateTime getLastStartTime() { - return lastStartTime; + return new DateTime(lastStartTimeInMillis); + } + + public long getLastStartTimeInMillis() + { + return lastStartTimeInMillis; } @Nullable @@ -290,7 +461,12 @@ public DateTime getLastStartTime() @ThriftField(4) public DateTime getLastEndTime() { - return lastEndTime; + return new DateTime(lastEndTimeInMillis); + } + + public long getLastEndTimeInMillis() + { + return lastEndTimeInMillis; } @Nullable @@ -298,7 +474,12 @@ public DateTime getLastEndTime() @ThriftField(5) public DateTime getEndTime() { - return endTime; + return new DateTime(endTimeInMillis); + } + + public long getEndTimeInMillis() + { + return endTimeInMillis; } @JsonProperty @@ -556,11 +737,11 @@ public RuntimeStats getRuntimeStats() public TaskStats summarize() { return new TaskStats( - createTime, - firstStartTime, - lastStartTime, - lastEndTime, - endTime, + new DateTime(createTimeInMillis), + new DateTime(firstStartTimeInMillis), + new DateTime(lastStartTimeInMillis), + new DateTime(lastEndTimeInMillis), + new DateTime(endTimeInMillis), elapsedTimeInNanos, queuedTimeInNanos, totalDrivers, @@ -602,11 +783,11 @@ public TaskStats summarize() public TaskStats summarizeFinal() { return new TaskStats( - createTime, - firstStartTime, - lastStartTime, - lastEndTime, - endTime, + new DateTime(createTimeInMillis), + new DateTime(firstStartTimeInMillis), + new DateTime(lastStartTimeInMillis), + new DateTime(lastEndTimeInMillis), + new DateTime(endTimeInMillis), elapsedTimeInNanos, queuedTimeInNanos, totalDrivers, diff --git a/presto-main/src/main/java/com/facebook/presto/server/BasicQueryStats.java b/presto-main/src/main/java/com/facebook/presto/server/BasicQueryStats.java index 22dbf5962c079..3da71c71af74d 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/BasicQueryStats.java +++ b/presto-main/src/main/java/com/facebook/presto/server/BasicQueryStats.java @@ -30,8 +30,10 @@ import java.util.OptionalDouble; import java.util.Set; +import static com.facebook.presto.util.DateTimeUtils.toTimeStampInMillis; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.units.DataSize.Unit.BYTE; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -43,8 +45,8 @@ @Immutable public class BasicQueryStats { - private final DateTime createTime; - private final DateTime endTime; + private final long createTimeInMillis; + private final long endTimeInMillis; private final Duration waitingForPrerequisitesTime; private final Duration queuedTime; @@ -81,41 +83,39 @@ public class BasicQueryStats private final OptionalDouble progressPercentage; - @ThriftConstructor - @JsonCreator public BasicQueryStats( - @JsonProperty("createTime") DateTime createTime, - @JsonProperty("endTime") DateTime endTime, - @JsonProperty("waitingForPrerequisitesTime") Duration waitingForPrerequisitesTime, - @JsonProperty("queuedTime") Duration queuedTime, - @JsonProperty("elapsedTime") Duration elapsedTime, - @JsonProperty("executionTime") Duration executionTime, - @JsonProperty("analysisTime") Duration analysisTime, - @JsonProperty("runningTasks") int runningTasks, - @JsonProperty("peakRunningTasks") int peakRunningTasks, - @JsonProperty("totalDrivers") int totalDrivers, - @JsonProperty("queuedDrivers") int queuedDrivers, - @JsonProperty("runningDrivers") int runningDrivers, - @JsonProperty("completedDrivers") int completedDrivers, - @JsonProperty("rawInputDataSize") DataSize rawInputDataSize, - @JsonProperty("rawInputPositions") long rawInputPositions, - @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, - @JsonProperty("cumulativeTotalMemory") double cumulativeTotalMemory, - @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, - @JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation, - @JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation, - @JsonProperty("peakTotalMemoryReservation") DataSize peakTotalMemoryReservation, - @JsonProperty("peakTaskTotalMemoryReservation") DataSize peakTaskTotalMemoryReservation, - @JsonProperty("peakNodeTotalMemoryReservation") DataSize peakNodeTotalMemoryReservation, - @JsonProperty("totalCpuTime") Duration totalCpuTime, - @JsonProperty("totalScheduledTime") Duration totalScheduledTime, - @JsonProperty("fullyBlocked") boolean fullyBlocked, - @JsonProperty("blockedReasons") Set blockedReasons, - @JsonProperty("totalAllocation") DataSize totalAllocation, - @JsonProperty("progressPercentage") OptionalDouble progressPercentage) - { - this.createTime = createTime; - this.endTime = endTime; + long createTimeInMillis, + long endTimeInMillis, + Duration waitingForPrerequisitesTime, + Duration queuedTime, + Duration elapsedTime, + Duration executionTime, + Duration analysisTime, + int runningTasks, + int peakRunningTasks, + int totalDrivers, + int queuedDrivers, + int runningDrivers, + int completedDrivers, + DataSize rawInputDataSize, + long rawInputPositions, + double cumulativeUserMemory, + double cumulativeTotalMemory, + DataSize userMemoryReservation, + DataSize totalMemoryReservation, + DataSize peakUserMemoryReservation, + DataSize peakTotalMemoryReservation, + DataSize peakTaskTotalMemoryReservation, + DataSize peakNodeTotalMemoryReservation, + Duration totalCpuTime, + Duration totalScheduledTime, + boolean fullyBlocked, + Set blockedReasons, + DataSize totalAllocation, + OptionalDouble progressPercentage) + { + this.createTimeInMillis = createTimeInMillis; + this.endTimeInMillis = endTimeInMillis; this.waitingForPrerequisitesTime = requireNonNull(waitingForPrerequisitesTime, "waitingForPrerequisitesTimex is null"); this.queuedTime = requireNonNull(queuedTime, "queuedTime is null"); @@ -156,10 +156,74 @@ public BasicQueryStats( this.progressPercentage = requireNonNull(progressPercentage, "progressPercentage is null"); } + @ThriftConstructor + @JsonCreator + public BasicQueryStats( + @JsonProperty("createTime") DateTime createTime, + @JsonProperty("endTime") DateTime endTime, + @JsonProperty("waitingForPrerequisitesTime") Duration waitingForPrerequisitesTime, + @JsonProperty("queuedTime") Duration queuedTime, + @JsonProperty("elapsedTime") Duration elapsedTime, + @JsonProperty("executionTime") Duration executionTime, + @JsonProperty("analysisTime") Duration analysisTime, + @JsonProperty("runningTasks") int runningTasks, + @JsonProperty("peakRunningTasks") int peakRunningTasks, + @JsonProperty("totalDrivers") int totalDrivers, + @JsonProperty("queuedDrivers") int queuedDrivers, + @JsonProperty("runningDrivers") int runningDrivers, + @JsonProperty("completedDrivers") int completedDrivers, + @JsonProperty("rawInputDataSize") DataSize rawInputDataSize, + @JsonProperty("rawInputPositions") long rawInputPositions, + @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, + @JsonProperty("cumulativeTotalMemory") double cumulativeTotalMemory, + @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, + @JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation, + @JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation, + @JsonProperty("peakTotalMemoryReservation") DataSize peakTotalMemoryReservation, + @JsonProperty("peakTaskTotalMemoryReservation") DataSize peakTaskTotalMemoryReservation, + @JsonProperty("peakNodeTotalMemoryReservation") DataSize peakNodeTotalMemoryReservation, + @JsonProperty("totalCpuTime") Duration totalCpuTime, + @JsonProperty("totalScheduledTime") Duration totalScheduledTime, + @JsonProperty("fullyBlocked") boolean fullyBlocked, + @JsonProperty("blockedReasons") Set blockedReasons, + @JsonProperty("totalAllocation") DataSize totalAllocation, + @JsonProperty("progressPercentage") OptionalDouble progressPercentage) + { + this(toTimeStampInMillis(createTime), + toTimeStampInMillis(endTime), + waitingForPrerequisitesTime, + queuedTime, + elapsedTime, + executionTime, + analysisTime, + runningTasks, + peakRunningTasks, + totalDrivers, + queuedDrivers, + runningDrivers, + completedDrivers, + rawInputDataSize, + rawInputPositions, + cumulativeUserMemory, + cumulativeTotalMemory, + userMemoryReservation, + totalMemoryReservation, + peakUserMemoryReservation, + peakTotalMemoryReservation, + peakTaskTotalMemoryReservation, + peakNodeTotalMemoryReservation, + totalCpuTime, + totalScheduledTime, + fullyBlocked, + blockedReasons, + totalAllocation, + progressPercentage); + } + public BasicQueryStats(QueryStats queryStats) { - this(queryStats.getCreateTime(), - queryStats.getEndTime(), + this(queryStats.getCreateTimeInMillis(), + queryStats.getEndTimeInMillis(), queryStats.getWaitingForPrerequisitesTime(), queryStats.getQueuedTime(), queryStats.getElapsedTime(), @@ -191,7 +255,7 @@ public BasicQueryStats(QueryStats queryStats) public static BasicQueryStats immediateFailureQueryStats() { - DateTime now = DateTime.now(); + long now = currentTimeMillis(); return new BasicQueryStats( now, now, @@ -228,14 +292,24 @@ public static BasicQueryStats immediateFailureQueryStats() @JsonProperty public DateTime getCreateTime() { - return createTime; + return new DateTime(createTimeInMillis); + } + + public long getCreateTimeInMillis() + { + return createTimeInMillis; } @ThriftField(2) @JsonProperty public DateTime getEndTime() { - return endTime; + return new DateTime(endTimeInMillis); + } + + public long getEndTimeInMillis() + { + return endTimeInMillis; } @ThriftField(3) diff --git a/presto-main/src/main/java/com/facebook/presto/server/QueryStateInfo.java b/presto-main/src/main/java/com/facebook/presto/server/QueryStateInfo.java index ff08f315cbc34..3619042d8b6c2 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/QueryStateInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/server/QueryStateInfo.java @@ -33,6 +33,7 @@ import static com.facebook.presto.execution.QueryState.QUEUED; import static com.facebook.presto.server.QueryProgressStats.createQueryProgressStats; +import static com.facebook.presto.util.DateTimeUtils.toTimeStampInMillis; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; @@ -44,7 +45,7 @@ public class QueryStateInfo private final Optional resourceGroupId; private final String query; private final boolean queryTruncated; - private final DateTime createTime; + private final long createTimeInMillis; private final String user; private final boolean authenticated; private final Optional source; @@ -56,6 +57,43 @@ public class QueryStateInfo private final List warningCodes; private final Optional errorCode; + public QueryStateInfo( + QueryId queryId, + QueryState queryState, + Optional resourceGroupId, + String query, + boolean queryTruncated, + long createTimeInMillis, + String user, + boolean authenticated, + Optional source, + Optional clientInfo, + Optional catalog, + Optional schema, + Optional> pathToRoot, + Optional progress, + List warningCodes, + Optional errorCode) + { + this.queryId = requireNonNull(queryId, "queryId is null"); + this.queryState = requireNonNull(queryState, "queryState is null"); + this.resourceGroupId = requireNonNull(resourceGroupId, "resourceGroupId is null"); + this.query = requireNonNull(query, "query text is null"); + this.queryTruncated = queryTruncated; + this.createTimeInMillis = requireNonNull(createTimeInMillis, "createTime is null"); + this.user = requireNonNull(user, "user is null"); + this.authenticated = authenticated; + this.source = requireNonNull(source, "source is null"); + this.clientInfo = requireNonNull(clientInfo, "clientInfo is null"); + this.catalog = requireNonNull(catalog, "catalog is null"); + this.schema = requireNonNull(schema, "schema is null"); + requireNonNull(pathToRoot, "pathToRoot is null"); + this.pathToRoot = pathToRoot.map(ImmutableList::copyOf); + this.progress = requireNonNull(progress, "progress is null"); + this.warningCodes = ImmutableList.copyOf(requireNonNull(warningCodes, "warningCodes is null")); + this.errorCode = requireNonNull(errorCode, "errorCode is null"); + } + @JsonCreator @ThriftConstructor public QueryStateInfo( @@ -76,23 +114,22 @@ public QueryStateInfo( @JsonProperty("warningCodes") List warningCodes, @JsonProperty("errorCode") Optional errorCode) { - this.queryId = requireNonNull(queryId, "queryId is null"); - this.queryState = requireNonNull(queryState, "queryState is null"); - this.resourceGroupId = requireNonNull(resourceGroupId, "resourceGroupId is null"); - this.query = requireNonNull(query, "query text is null"); - this.queryTruncated = queryTruncated; - this.createTime = requireNonNull(createTime, "createTime is null"); - this.user = requireNonNull(user, "user is null"); - this.authenticated = authenticated; - this.source = requireNonNull(source, "source is null"); - this.clientInfo = requireNonNull(clientInfo, "clientInfo is null"); - this.catalog = requireNonNull(catalog, "catalog is null"); - this.schema = requireNonNull(schema, "schema is null"); - requireNonNull(pathToRoot, "pathToRoot is null"); - this.pathToRoot = pathToRoot.map(ImmutableList::copyOf); - this.progress = requireNonNull(progress, "progress is null"); - this.warningCodes = ImmutableList.copyOf(requireNonNull(warningCodes, "warningCodes is null")); - this.errorCode = requireNonNull(errorCode, "errorCode is null"); + this(queryId, + queryState, + resourceGroupId, + query, + queryTruncated, + toTimeStampInMillis(createTime), + user, + authenticated, + source, + clientInfo, + catalog, + schema, + pathToRoot, + progress, + warningCodes, + errorCode); } public static QueryStateInfo createQueryStateInfo(BasicQueryInfo queryInfo) @@ -133,7 +170,7 @@ private static QueryStateInfo createQueryStateInfo( queryInfo.getResourceGroupId(), query, queryTruncated, - queryInfo.getQueryStats().getCreateTime(), + queryInfo.getQueryStats().getCreateTimeInMillis(), queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal().isPresent(), queryInfo.getSession().getSource(), @@ -234,7 +271,12 @@ public Optional> getPathToRoot() @ThriftField(13) public DateTime getCreateTime() { - return createTime; + return new DateTime(createTimeInMillis); + } + + public long getCreateTimeInMillis() + { + return createTimeInMillis; } @JsonProperty diff --git a/presto-main/src/main/java/com/facebook/presto/server/TaskResourceUtils.java b/presto-main/src/main/java/com/facebook/presto/server/TaskResourceUtils.java index 688af76987feb..409bc41e1b025 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/TaskResourceUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/server/TaskResourceUtils.java @@ -54,7 +54,7 @@ public static TaskInfo convertToThriftTaskInfo( return new TaskInfo( taskInfo.getTaskId(), taskInfo.getTaskStatus(), - taskInfo.getLastHeartbeat(), + taskInfo.getLastHeartbeatInMillis(), taskInfo.getOutputBuffers(), taskInfo.getNoMoreSplits(), convertToThriftTaskStats(taskInfo.getStats()), @@ -70,11 +70,11 @@ private static TaskStats convertToThriftTaskStats(TaskStats taskStats) } return new TaskStats( - taskStats.getCreateTime(), - taskStats.getFirstStartTime(), - taskStats.getLastStartTime(), - taskStats.getLastEndTime(), - taskStats.getEndTime(), + taskStats.getCreateTimeInMillis(), + taskStats.getFirstStartTimeInMillis(), + taskStats.getLastStartTimeInMillis(), + taskStats.getLastEndTimeInMillis(), + taskStats.getEndTimeInMillis(), taskStats.getElapsedTimeInNanos(), taskStats.getQueuedTimeInNanos(), taskStats.getTotalDrivers(), @@ -128,9 +128,9 @@ private static PipelineStats convertToThriftPipelineStats(PipelineStats pipeline return new PipelineStats( pipelineStats.getPipelineId(), - pipelineStats.getFirstStartTime(), - pipelineStats.getLastStartTime(), - pipelineStats.getLastEndTime(), + pipelineStats.getFirstStartTimeInMillis(), + pipelineStats.getLastStartTimeInMillis(), + pipelineStats.getLastEndTimeInMillis(), pipelineStats.isInputPipeline(), pipelineStats.isOutputPipeline(), pipelineStats.getTotalDrivers(), @@ -175,9 +175,9 @@ private static DriverStats convertToThriftDriverStats(DriverStats driverStats) { return new DriverStats( driverStats.getLifespan(), - driverStats.getCreateTime(), - driverStats.getStartTime(), - driverStats.getEndTime(), + driverStats.getCreateTimeInMillis(), + driverStats.getStartTimeInMillis(), + driverStats.getEndTimeInMillis(), driverStats.getQueuedTime(), driverStats.getElapsedTime(), driverStats.getUserMemoryReservationInBytes(), @@ -293,7 +293,7 @@ public static TaskInfo convertFromThriftTaskInfo( return new TaskInfo( thriftTaskInfo.getTaskId(), thriftTaskInfo.getTaskStatus(), - thriftTaskInfo.getLastHeartbeat(), + thriftTaskInfo.getLastHeartbeatInMillis(), thriftTaskInfo.getOutputBuffers(), thriftTaskInfo.getNoMoreSplits(), convertFromThriftTaskStats(thriftTaskInfo.getStats()), @@ -309,11 +309,11 @@ private static TaskStats convertFromThriftTaskStats(TaskStats thriftTaskStats) } return new TaskStats( - thriftTaskStats.getCreateTime(), - thriftTaskStats.getFirstStartTime(), - thriftTaskStats.getLastStartTime(), - thriftTaskStats.getLastEndTime(), - thriftTaskStats.getEndTime(), + thriftTaskStats.getCreateTimeInMillis(), + thriftTaskStats.getFirstStartTimeInMillis(), + thriftTaskStats.getLastStartTimeInMillis(), + thriftTaskStats.getLastEndTimeInMillis(), + thriftTaskStats.getEndTimeInMillis(), thriftTaskStats.getElapsedTimeInNanos(), thriftTaskStats.getQueuedTimeInNanos(), thriftTaskStats.getTotalDrivers(), @@ -367,9 +367,9 @@ private static PipelineStats convertFromThriftPipelineStats(PipelineStats thrift return new PipelineStats( thriftPipelineStats.getPipelineId(), - thriftPipelineStats.getFirstStartTime(), - thriftPipelineStats.getLastStartTime(), - thriftPipelineStats.getLastEndTime(), + thriftPipelineStats.getFirstStartTimeInMillis(), + thriftPipelineStats.getLastStartTimeInMillis(), + thriftPipelineStats.getLastEndTimeInMillis(), thriftPipelineStats.isInputPipeline(), thriftPipelineStats.isOutputPipeline(), thriftPipelineStats.getTotalDrivers(), @@ -414,9 +414,9 @@ private static DriverStats convertFromThriftDriverStats(DriverStats thriftDriver { return new DriverStats( thriftDriverStats.getLifespan(), - thriftDriverStats.getCreateTime(), - thriftDriverStats.getStartTime(), - thriftDriverStats.getEndTime(), + thriftDriverStats.getCreateTimeInMillis(), + thriftDriverStats.getStartTimeInMillis(), + thriftDriverStats.getEndTimeInMillis(), thriftDriverStats.getQueuedTime(), thriftDriverStats.getElapsedTime(), thriftDriverStats.getUserMemoryReservationInBytes(), diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java index 0fbb291a6ca68..36ec0c54c850f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java @@ -77,7 +77,6 @@ import com.sun.management.ThreadMXBean; import io.airlift.units.Duration; import it.unimi.dsi.fastutil.longs.LongArrayList; -import org.joda.time.DateTime; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; @@ -137,6 +136,7 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.lang.Math.addExact; import static java.lang.String.format; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -359,7 +359,7 @@ public HttpRemoteTask( .map(outputId -> new BufferInfo(outputId, false, 0, 0, PageBufferInfo.empty())) .collect(toImmutableList()); - TaskInfo initialTask = createInitialTask(taskId, location, bufferStates, new TaskStats(DateTime.now(), null), nodeId); + TaskInfo initialTask = createInitialTask(taskId, location, bufferStates, new TaskStats(currentTimeMillis(), 0), nodeId); this.taskStatusFetcher = new ContinuousTaskStatusFetcher( this::failTask, diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskWithEventLoop.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskWithEventLoop.java index 5de5a3508671c..51aae8a06c5d4 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskWithEventLoop.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskWithEventLoop.java @@ -78,7 +78,6 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; import it.unimi.dsi.fastutil.longs.LongArrayList; -import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -132,6 +131,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static java.lang.Math.addExact; import static java.lang.String.format; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -433,7 +433,7 @@ private HttpRemoteTaskWithEventLoop(Session session, .map(outputId -> new BufferInfo(outputId, false, 0, 0, PageBufferInfo.empty())) .collect(toImmutableList()); - TaskInfo initialTask = createInitialTask(taskId, location, bufferStates, new TaskStats(DateTime.now(), null), nodeId); + TaskInfo initialTask = createInitialTask(taskId, location, bufferStates, new TaskStats(currentTimeMillis(), 0), nodeId); this.taskStatusFetcher = new ContinuousTaskStatusFetcherWithEventLoop( this::failTask, diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java b/presto-main/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java index 6b75493cedb8b..1378b30e407f6 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java @@ -35,7 +35,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import io.airlift.units.Duration; -import org.joda.time.DateTime; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -75,6 +74,7 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import static java.lang.String.format; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.stream.Collectors.toList; @@ -358,7 +358,7 @@ public void fail(TransactionId transactionId) @ThreadSafe private static class TransactionMetadata { - private final DateTime createTime = DateTime.now(); + private final long createTimeInMillis = currentTimeMillis(); private final CatalogManager catalogManager; private final TransactionId transactionId; private final IsolationLevel isolationLevel; @@ -633,10 +633,10 @@ private synchronized ListenableFuture abortInternal() { // the callbacks in statement performed on another thread so are safe List> abortFutures = Stream.concat( - functionNamespaceTransactions.values().stream() - .map(transactionMetadata -> finishingExecutor.submit(() -> safeAbort(transactionMetadata))), - connectorIdToMetadata.values().stream() - .map(connection -> finishingExecutor.submit(() -> safeAbort(connection)))) + functionNamespaceTransactions.values().stream() + .map(transactionMetadata -> finishingExecutor.submit(() -> safeAbort(transactionMetadata))), + connectorIdToMetadata.values().stream() + .map(connection -> finishingExecutor.submit(() -> safeAbort(connection)))) .collect(toList()); return nonCancellationPropagating(Futures.allAsList(abortFutures)); } @@ -673,7 +673,7 @@ public TransactionInfo getTransactionInfo() // copying the key set is safe here because the map is concurrent @SuppressWarnings("FieldAccessNotGuarded") List connectorIds = ImmutableList.copyOf(connectorIdToMetadata.keySet()); - return new TransactionInfo(transactionId, isolationLevel, readOnly, autoCommitContext, createTime, idleTime, connectorIds, writtenConnectorId); + return new TransactionInfo(transactionId, isolationLevel, readOnly, autoCommitContext, createTimeInMillis, idleTime, connectorIds, writtenConnectorId); } private static class ConnectorTransactionMetadata diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionInfo.java b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionInfo.java index 5411d05d011b8..1a3999a4977cc 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionInfo.java @@ -21,11 +21,11 @@ import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.collect.ImmutableList; import io.airlift.units.Duration; -import org.joda.time.DateTime; import java.util.List; import java.util.Optional; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @ThriftStruct @@ -35,7 +35,7 @@ public class TransactionInfo private final IsolationLevel isolationLevel; private final boolean readOnly; private final boolean autoCommitContext; - private final DateTime createTime; + private final long createTimeInMillis; private final Duration idleTime; private final List connectorIds; private final Optional writtenConnectorId; @@ -46,7 +46,7 @@ public TransactionInfo( IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext, - DateTime createTime, + long createTimeInMillis, Duration idleTime, List connectorIds, Optional writtenConnectorId) @@ -55,7 +55,8 @@ public TransactionInfo( this.isolationLevel = requireNonNull(isolationLevel, "isolationLevel is null"); this.readOnly = readOnly; this.autoCommitContext = autoCommitContext; - this.createTime = requireNonNull(createTime, "createTime is null"); + checkArgument(createTimeInMillis >= 0, "createTimeInMillis is negative"); + this.createTimeInMillis = createTimeInMillis; this.idleTime = requireNonNull(idleTime, "idleTime is null"); this.connectorIds = ImmutableList.copyOf(requireNonNull(connectorIds, "connectorIds is null")); this.writtenConnectorId = requireNonNull(writtenConnectorId, "writtenConnectorId is null"); @@ -86,9 +87,9 @@ public boolean isAutoCommitContext() } @ThriftField(5) - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return createTime; + return createTimeInMillis; } @ThriftField(6) diff --git a/presto-main/src/main/java/com/facebook/presto/util/DateTimeUtils.java b/presto-main/src/main/java/com/facebook/presto/util/DateTimeUtils.java index f33cf36076f94..a6b63679c2699 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/DateTimeUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/util/DateTimeUtils.java @@ -36,6 +36,8 @@ import org.joda.time.format.PeriodFormatterBuilder; import org.joda.time.format.PeriodParser; +import javax.annotation.Nullable; + import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.reflect.Method; @@ -153,6 +155,14 @@ public static String printDate(int days) } } + public static long toTimeStampInMillis(@Nullable DateTime dateTime) + { + if (dateTime == null) { + return 0L; + } + return dateTime.getMillis(); + } + /** * Parse a string (optionally containing a zone) as a value of either TIMESTAMP or TIMESTAMP WITH TIME ZONE type. *

diff --git a/presto-main/src/main/java/com/facebook/presto/util/GcStatusMonitor.java b/presto-main/src/main/java/com/facebook/presto/util/GcStatusMonitor.java index e427421da5fb1..15d5083945470 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/GcStatusMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/util/GcStatusMonitor.java @@ -178,12 +178,12 @@ private static void logQueriesAndTasks(List queryIds, ListMultimap>builder().add( - ImmutableList.of( - "Query ID", - "Total Memory Reservation", - "User Memory Reservation", - "System Memory Reservation", - "Revocable Memory Reservation")) + ImmutableList.of( + "Query ID", + "Total Memory Reservation", + "User Memory Reservation", + "System Memory Reservation", + "Revocable Memory Reservation")) .addAll(rows) .build()); } @@ -210,7 +210,7 @@ private static void logTaskStats(List queryIds, ListMultimap getExecutionStartTime() + public long getExecutionStartTimeInMillis() { - return Optional.empty(); + return 0L; } @Override - public DateTime getLastHeartbeat() + public long getLastHeartbeatInMillis() { - return null; + return 0L; } @Override - public Optional getEndTime() + public long getEndTimeInMillis() { - return Optional.empty(); + return 0L; } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java index 5ae89b3f2716c..94f18c5efc404 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java @@ -59,7 +59,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import io.airlift.units.DataSize; -import org.joda.time.DateTime; import javax.annotation.concurrent.GuardedBy; @@ -95,6 +94,7 @@ import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.Math.addExact; +import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; public class MockRemoteTaskFactory @@ -294,10 +294,10 @@ public TaskInfo getTaskInfo() 0, 0, 0, - System.currentTimeMillis() + 100 - stats.getCreateTime().getMillis(), + currentTimeMillis() + 100 - stats.getCreateTimeInMillis(), 0L, 0L), - DateTime.now(), + currentTimeMillis(), outputBuffer.getInfo(), ImmutableSet.of(), taskContext.getTaskStats(), @@ -338,7 +338,7 @@ public TaskStatus getTaskStatus() 0, stats.getTotalCpuTimeInNanos(), // Adding 100 millis to make sure task age > 0 for testing - System.currentTimeMillis() + 100 - stats.getCreateTime().getMillis(), + currentTimeMillis() + 100 - stats.getCreateTimeInMillis(), queuedSplitsInfo.getWeightSum(), combinedSplitsInfo.getWeightSum() - queuedSplitsInfo.getWeightSum()); } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java index 3eb21928e9cba..93ef33c6e797b 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java @@ -177,8 +177,8 @@ private BasicQueryStats getBasicQueryStats(Duration executionTime) { Duration defaultDuration = Duration.valueOf("5m"); return new BasicQueryStats( - null, - null, + 0, + 0, defaultDuration, defaultDuration, defaultDuration, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java index 036346750516e..8172ce296a0c9 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java @@ -573,18 +573,18 @@ private static void assertState(QueryStateMachine stateMachine, QueryState expec assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getFinishingTime()); - assertNotNull(queryStats.getCreateTime()); + assertTrue(queryStats.getCreateTimeInMillis() > 0); if (queryInfo.getState() == WAITING_FOR_PREREQUISITES || queryInfo.getState() == QUEUED || queryInfo.getState() == WAITING_FOR_RESOURCES || queryInfo.getState() == DISPATCHING) { - assertNull(queryStats.getExecutionStartTime()); + assertEquals(queryStats.getExecutionStartTimeInMillis(), 0L); } else { - assertNotNull(queryStats.getExecutionStartTime()); + assertTrue(queryStats.getExecutionStartTimeInMillis() > 0); } if (queryInfo.getState().isDone()) { - assertNotNull(queryStats.getEndTime()); + assertTrue(queryStats.getEndTimeInMillis() > 0); } else { - assertNull(queryStats.getEndTime()); + assertEquals(queryStats.getEndTimeInMillis(), 0); } assertEquals(stateMachine.getQueryState(), expectedState); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java index decf70fcd04fc..340f8737fe78d 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java @@ -208,10 +208,10 @@ public class TestQueryStats 0)); static final QueryStats EXPECTED = new QueryStats( - new DateTime(1), - new DateTime(2), - new DateTime(3), - new DateTime(4), + 1L, + 2L, + 3L, + 4L, new Duration(6, NANOSECONDS), new Duration(7, NANOSECONDS), new Duration(5, NANOSECONDS), @@ -434,10 +434,10 @@ public void testJson() static void assertExpectedQueryStats(QueryStats actual) { - assertEquals(actual.getCreateTime(), new DateTime(1, UTC)); - assertEquals(actual.getExecutionStartTime(), new DateTime(2, UTC)); - assertEquals(actual.getLastHeartbeat(), new DateTime(3, UTC)); - assertEquals(actual.getEndTime(), new DateTime(4, UTC)); + assertEquals(actual.getCreateTimeInMillis(), new DateTime(1, UTC).getMillis()); + assertEquals(actual.getExecutionStartTimeInMillis(), new DateTime(2, UTC).getMillis()); + assertEquals(actual.getLastHeartbeatInMillis(), new DateTime(3, UTC).getMillis()); + assertEquals(actual.getEndTimeInMillis(), new DateTime(4, UTC).getMillis()); assertEquals(actual.getElapsedTime(), new Duration(6, NANOSECONDS)); assertEquals(actual.getQueuedTime(), new Duration(5, NANOSECONDS)); @@ -583,7 +583,7 @@ private static StageExecutionStats createStageStats(int stageId, int stageExecut List operatorSummaries) { return new StageExecutionStats( - new DateTime(0), + 0L, new Distribution(0).snapshot(), diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java index ebc4f37aa2768..ff737c3567901 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java @@ -71,8 +71,6 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -205,19 +203,19 @@ public void testCancel() .withNoMoreBufferIds(), Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); - assertNull(taskInfo.getStats().getEndTime()); + assertEquals(taskInfo.getStats().getEndTimeInMillis(), 0); taskInfo = sqlTask.getTaskInfo(); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); - assertNull(taskInfo.getStats().getEndTime()); + assertEquals(taskInfo.getStats().getEndTimeInMillis(), 0); taskInfo = sqlTask.cancel(); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED); - assertNotNull(taskInfo.getStats().getEndTime()); + assertTrue(taskInfo.getStats().getEndTimeInMillis() > 0); taskInfo = sqlTask.getTaskInfo(); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED); - assertNotNull(taskInfo.getStats().getEndTime()); + assertTrue(taskInfo.getStats().getEndTimeInMillis() > 0); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java index 79669ede8b287..2a7d731e59df4 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java @@ -69,8 +69,6 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @Test(singleThreaded = true) @@ -197,19 +195,19 @@ public void testCancel() TaskId taskId = TASK_ID; TaskInfo taskInfo = createTask(sqlTaskManager, taskId, createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); - assertNull(taskInfo.getStats().getEndTime()); + assertEquals(taskInfo.getStats().getEndTimeInMillis(), 0); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); - assertNull(taskInfo.getStats().getEndTime()); + assertEquals(taskInfo.getStats().getEndTimeInMillis(), 0); taskInfo = sqlTaskManager.cancelTask(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED); - assertNotNull(taskInfo.getStats().getEndTime()); + assertTrue(taskInfo.getStats().getEndTimeInMillis() > 0); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED); - assertNotNull(taskInfo.getStats().getEndTime()); + assertTrue(taskInfo.getStats().getEndTimeInMillis() > 0); } } @@ -220,19 +218,19 @@ public void testAbort() TaskId taskId = TASK_ID; TaskInfo taskInfo = createTask(sqlTaskManager, taskId, createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); - assertNull(taskInfo.getStats().getEndTime()); + assertEquals(taskInfo.getStats().getEndTimeInMillis(), 0); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); - assertNull(taskInfo.getStats().getEndTime()); + assertEquals(taskInfo.getStats().getEndTimeInMillis(), 0); taskInfo = sqlTaskManager.abortTask(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.ABORTED); - assertNotNull(taskInfo.getStats().getEndTime()); + assertTrue(taskInfo.getStats().getEndTimeInMillis() > 0); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.ABORTED); - assertNotNull(taskInfo.getStats().getEndTime()); + assertTrue(taskInfo.getStats().getEndTimeInMillis() > 0); } } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStats.java b/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStats.java index 86387ce04bf49..fd477c0a3ea9d 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStats.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStats.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import io.airlift.units.Duration; -import org.joda.time.DateTime; import org.testng.annotations.Test; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -30,7 +29,7 @@ public class TestStageExecutionStats { private static final StageExecutionStats EXPECTED = new StageExecutionStats( - new DateTime(0), + 0L, getTestDistribution(1), @@ -101,7 +100,7 @@ public void testJson() private static void assertExpectedStageStats(StageExecutionStats actual) { - assertEquals(actual.getSchedulingComplete().getMillis(), 0); + assertEquals(actual.getSchedulingCompleteInMillis(), 0); assertEquals(actual.getGetSplitDistribution().getCount(), 1.0); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestThriftResourceGroupInfo.java b/presto-main/src/test/java/com/facebook/presto/execution/TestThriftResourceGroupInfo.java index e1d7c3fae93a6..ee18213a7501e 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestThriftResourceGroupInfo.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestThriftResourceGroupInfo.java @@ -291,7 +291,7 @@ private void setUpQueryStateInfoList() Optional.of(FAKE_QUERY_RESOURCE_GROUP_ID), FAKE_QUERY, FAKE_QUERY_TRUNCATED, - FAKE_CREATE_TIME, + FAKE_CREATE_TIME.getMillis(), FAKE_QUERY_REQUESTER, FAKE_AUTHENTICATED, Optional.of(FAKE_QUERY_SOURCE), @@ -308,7 +308,7 @@ private void setUpQueryStateInfoList() Optional.of(FAKE_QUERY_RESOURCE_GROUP_ID), FAKE_QUERY, FAKE_QUERY_TRUNCATED, - FAKE_CREATE_TIME, + FAKE_CREATE_TIME.getMillis(), FAKE_QUERY_REQUESTER, FAKE_AUTHENTICATED, Optional.of(FAKE_QUERY_SOURCE), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestDriverStats.java b/presto-main/src/test/java/com/facebook/presto/operator/TestDriverStats.java index e0b2545e8814d..5dd790b5384d8 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestDriverStats.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestDriverStats.java @@ -31,9 +31,9 @@ public class TestDriverStats public static final DriverStats EXPECTED = new DriverStats( Lifespan.driverGroup(21), - new DateTime(1), - new DateTime(2), - new DateTime(3), + 1L, + 2L, + 3L, new Duration(4, NANOSECONDS), new Duration(5, NANOSECONDS), @@ -79,9 +79,9 @@ public static void assertExpectedDriverStats(DriverStats actual) { assertEquals(actual.getLifespan(), Lifespan.driverGroup(21)); - assertEquals(actual.getCreateTime(), new DateTime(1, UTC)); - assertEquals(actual.getStartTime(), new DateTime(2, UTC)); - assertEquals(actual.getEndTime(), new DateTime(3, UTC)); + assertEquals(actual.getCreateTimeInMillis(), new DateTime(1, UTC).getMillis()); + assertEquals(actual.getStartTimeInMillis(), new DateTime(2, UTC).getMillis()); + assertEquals(actual.getEndTimeInMillis(), new DateTime(3, UTC).getMillis()); assertEquals(actual.getQueuedTime(), new Duration(4, NANOSECONDS)); assertEquals(actual.getElapsedTime(), new Duration(5, NANOSECONDS)); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorInfoUnionSerde.java b/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorInfoUnionSerde.java index 703b90dba666d..0fad61470be2d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorInfoUnionSerde.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorInfoUnionSerde.java @@ -149,7 +149,7 @@ private ExchangeClientStatus getExchangeClientStatus() ImmutableList.of(new PageBufferClientStatus( URI.create("http://fake"), "running", - new DateTime(2022, 10, 28, 16, 7, 15, 0), + new DateTime(2022, 10, 28, 16, 7, 15, 0).getMillis(), 7174L, 612, OptionalLong.of(93L), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestPipelineStats.java b/presto-main/src/test/java/com/facebook/presto/operator/TestPipelineStats.java index c331ffc6aeeb5..03482c9e55535 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestPipelineStats.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestPipelineStats.java @@ -31,9 +31,9 @@ public class TestPipelineStats public static final PipelineStats EXPECTED = new PipelineStats( 0, - new DateTime(100), - new DateTime(101), - new DateTime(102), + 100L, + 101L, + 102L, true, false, @@ -90,9 +90,9 @@ public void testJson() public static void assertExpectedPipelineStats(PipelineStats actual) { - assertEquals(actual.getFirstStartTime(), new DateTime(100, UTC)); - assertEquals(actual.getLastStartTime(), new DateTime(101, UTC)); - assertEquals(actual.getLastEndTime(), new DateTime(102, UTC)); + assertEquals(actual.getFirstStartTimeInMillis(), new DateTime(100, UTC).getMillis()); + assertEquals(actual.getLastStartTimeInMillis(), new DateTime(101, UTC).getMillis()); + assertEquals(actual.getLastEndTimeInMillis(), new DateTime(102, UTC).getMillis()); assertEquals(actual.isInputPipeline(), true); assertEquals(actual.isOutputPipeline(), false); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTaskStats.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTaskStats.java index 3c6815aabc6fa..775ca3bd9c91c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTaskStats.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTaskStats.java @@ -27,11 +27,11 @@ public class TestTaskStats { private static final TaskStats EXPECTED = new TaskStats( - new DateTime(1), - new DateTime(2), - new DateTime(100), - new DateTime(101), - new DateTime(3), + 1L, + 2L, + 100L, + 101L, + 3L, 4, 5, @@ -91,11 +91,11 @@ public void testJson() public static void assertExpectedTaskStats(TaskStats actual) { - assertEquals(actual.getCreateTime(), new DateTime(1, UTC)); - assertEquals(actual.getFirstStartTime(), new DateTime(2, UTC)); - assertEquals(actual.getLastStartTime(), new DateTime(100, UTC)); - assertEquals(actual.getLastEndTime(), new DateTime(101, UTC)); - assertEquals(actual.getEndTime(), new DateTime(3, UTC)); + assertEquals(actual.getCreateTimeInMillis(), new DateTime(1, UTC).getMillis()); + assertEquals(actual.getFirstStartTimeInMillis(), new DateTime(2, UTC).getMillis()); + assertEquals(actual.getLastStartTimeInMillis(), new DateTime(100, UTC).getMillis()); + assertEquals(actual.getLastEndTimeInMillis(), new DateTime(101, UTC).getMillis()); + assertEquals(actual.getEndTimeInMillis(), new DateTime(3, UTC).getMillis()); assertEquals(actual.getElapsedTimeInNanos(), 4); assertEquals(actual.getQueuedTimeInNanos(), 5); diff --git a/presto-main/src/test/java/com/facebook/presto/resourcemanager/TestResourceManagerClusterStateProvider.java b/presto-main/src/test/java/com/facebook/presto/resourcemanager/TestResourceManagerClusterStateProvider.java index 2e755295eae5e..29801427f6982 100644 --- a/presto-main/src/test/java/com/facebook/presto/resourcemanager/TestResourceManagerClusterStateProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/resourcemanager/TestResourceManagerClusterStateProvider.java @@ -747,8 +747,8 @@ private static BasicQueryInfo createQueryInfo(String queryId, QueryState state, URI.create("1"), "", new BasicQueryStats( - DateTime.parse("1991-09-06T05:00-05:30"), - DateTime.parse("1991-09-06T05:01-05:30"), + new DateTime("1991-09-06T05:00").getMillis(), + new DateTime("1991-09-06T05:01").getMillis(), Duration.valueOf("6m"), Duration.valueOf("8m"), Duration.valueOf("7m"), diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java b/presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java index f52803f7db7d1..b30ce4c921f04 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java @@ -57,10 +57,10 @@ public void testConstructor() Optional.empty(), Optional.empty(), new QueryStats( - DateTime.parse("1991-09-06T05:00-05:30"), - DateTime.parse("1991-09-06T05:01-05:30"), - DateTime.parse("1991-09-06T05:02-05:30"), - DateTime.parse("1991-09-06T06:00-05:30"), + new DateTime("1991-09-06T05:00").getMillis(), + new DateTime("1991-09-06T05:01").getMillis(), + new DateTime("1991-09-06T05:02").getMillis(), + new DateTime("1991-09-06T06:00").getMillis(), Duration.valueOf("8m"), Duration.valueOf("5m"), Duration.valueOf("7m"), @@ -162,8 +162,8 @@ public void testConstructor() assertEquals(basicInfo.getQueryHash(), "235ca215765cbfca"); assertEquals(basicInfo.getQueryType(), Optional.of(QueryType.INSERT)); - assertEquals(basicInfo.getQueryStats().getCreateTime(), DateTime.parse("1991-09-06T05:00-05:30")); - assertEquals(basicInfo.getQueryStats().getEndTime(), DateTime.parse("1991-09-06T06:00-05:30")); + assertEquals(basicInfo.getQueryStats().getCreateTime(), new DateTime("1991-09-06T05:00")); + assertEquals(basicInfo.getQueryStats().getEndTime(), new DateTime("1991-09-06T06:00")); assertEquals(basicInfo.getQueryStats().getElapsedTime(), Duration.valueOf("8m")); assertEquals(basicInfo.getQueryStats().getExecutionTime(), Duration.valueOf("44m")); assertEquals(basicInfo.getQueryStats().getWaitingForPrerequisitesTime(), Duration.valueOf("5m")); diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java b/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java index 98ae1d383c377..8bdc2b62aec0e 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java @@ -164,7 +164,7 @@ public void testQueryStateInfoCreation() assertEquals(queryStateInfo.getQueryState(), queryInfo.getState()); assertEquals(queryStateInfo.getResourceGroupId(), queryInfo.getResourceGroupId()); assertEquals(queryStateInfo.getQuery(), queryInfo.getQuery()); - assertEquals(queryStateInfo.getCreateTime(), queryInfo.getQueryStats().getCreateTime()); + assertEquals(queryStateInfo.getCreateTime().getMillis(), queryInfo.getQueryStats().getCreateTimeInMillis()); assertEquals(queryStateInfo.getUser(), queryInfo.getSession().getUser()); assertEquals(queryStateInfo.isAuthenticated(), queryInfo.getSession().getPrincipal().isPresent()); assertEquals(queryStateInfo.getSource(), queryInfo.getSession().getSource()); @@ -217,10 +217,10 @@ private QueryInfo createQueryInfo(String queryId, ResourceGroupId resourceGroupI Optional.empty(), Optional.empty(), new QueryStats( - DateTime.parse("1991-09-06T05:00-05:30"), - DateTime.parse("1991-09-06T05:01-05:30"), - DateTime.parse("1991-09-06T05:02-05:30"), - DateTime.parse("1991-09-06T06:00-05:30"), + new DateTime("1991-09-06T05:00").getMillis(), + new DateTime("1991-09-06T05:01").getMillis(), + new DateTime("1991-09-06T05:02").getMillis(), + new DateTime("1991-09-06T06:00").getMillis(), Duration.valueOf("8m"), Duration.valueOf("5m"), Duration.valueOf("7m"), diff --git a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java index f2a01abea2a9c..81f888b9647b4 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java +++ b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java @@ -604,7 +604,7 @@ private TaskInfo buildTaskInfo() return new TaskInfo( initialTaskInfo.getTaskId(), buildTaskStatus(), - initialTaskInfo.getLastHeartbeat(), + initialTaskInfo.getLastHeartbeatInMillis(), initialTaskInfo.getOutputBuffers(), initialTaskInfo.getNoMoreSplits(), initialTaskInfo.getStats(), diff --git a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTaskWithEventLoop.java b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTaskWithEventLoop.java index 92883d2202177..517da0b593fb4 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTaskWithEventLoop.java +++ b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTaskWithEventLoop.java @@ -604,7 +604,7 @@ private TaskInfo buildTaskInfo() return new TaskInfo( initialTaskInfo.getTaskId(), buildTaskStatus(), - initialTaskInfo.getLastHeartbeat(), + initialTaskInfo.getLastHeartbeatInMillis(), initialTaskInfo.getOutputBuffers(), initialTaskInfo.getNoMoreSplits(), initialTaskInfo.getStats(), diff --git a/presto-native-execution/presto_cpp/main/PrestoTask.cpp b/presto-native-execution/presto_cpp/main/PrestoTask.cpp index fc000954e082a..b169bcdbd8667 100644 --- a/presto-native-execution/presto_cpp/main/PrestoTask.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoTask.cpp @@ -313,9 +313,9 @@ void updatePipelineStats( protocol::PipelineStats& prestoPipelineStats) { prestoPipelineStats.inputPipeline = veloxPipelineStats.inputPipeline; prestoPipelineStats.outputPipeline = veloxPipelineStats.outputPipeline; - prestoPipelineStats.firstStartTime = prestoTaskStats.createTime; - prestoPipelineStats.lastStartTime = prestoTaskStats.endTime; - prestoPipelineStats.lastEndTime = prestoTaskStats.endTime; + prestoPipelineStats.firstStartTimeInMillis = prestoTaskStats.createTimeInMillis; + prestoPipelineStats.lastStartTimeInMillis = prestoTaskStats.endTimeInMillis; + prestoPipelineStats.lastEndTimeInMillis = prestoTaskStats.endTimeInMillis; prestoPipelineStats.operatorSummaries.resize( veloxPipelineStats.operatorStats.size()); @@ -468,7 +468,7 @@ PrestoTask::PrestoTask( void PrestoTask::updateHeartbeatLocked() { lastHeartbeatMs = velox::getCurrentTimeMs(); - info.lastHeartbeat = util::toISOTimestamp(lastHeartbeatMs); + info.lastHeartbeatInMillis = lastHeartbeatMs; } void PrestoTask::updateCoordinatorHeartbeat() { @@ -691,18 +691,13 @@ void PrestoTask::updateTimeInfoLocked( prestoTaskStats.totalCpuTimeInNanos = {}; prestoTaskStats.totalBlockedTimeInNanos = {}; - prestoTaskStats.createTime = - util::toISOTimestamp(veloxTaskStats.executionStartTimeMs); - prestoTaskStats.firstStartTime = - util::toISOTimestamp(veloxTaskStats.firstSplitStartTimeMs); + prestoTaskStats.createTimeInMillis = veloxTaskStats.executionStartTimeMs; + prestoTaskStats.firstStartTimeInMillis = veloxTaskStats.firstSplitStartTimeMs; createTimeMs = veloxTaskStats.executionStartTimeMs; firstSplitStartTimeMs = veloxTaskStats.firstSplitStartTimeMs; - prestoTaskStats.lastStartTime = - util::toISOTimestamp(veloxTaskStats.lastSplitStartTimeMs); - prestoTaskStats.lastEndTime = - util::toISOTimestamp(veloxTaskStats.executionEndTimeMs); - prestoTaskStats.endTime = - util::toISOTimestamp(veloxTaskStats.executionEndTimeMs); + prestoTaskStats.lastStartTimeInMillis = veloxTaskStats.lastSplitStartTimeMs; + prestoTaskStats.lastEndTimeInMillis = veloxTaskStats.executionEndTimeMs; + prestoTaskStats.endTimeInMillis = veloxTaskStats.executionEndTimeMs; lastEndTimeMs = veloxTaskStats.executionEndTimeMs; if (veloxTaskStats.executionEndTimeMs > veloxTaskStats.executionStartTimeMs) { diff --git a/presto-native-execution/presto_cpp/main/TaskManager.cpp b/presto-native-execution/presto_cpp/main/TaskManager.cpp index 55e9a6709b8cc..817129d3eaaf7 100644 --- a/presto-native-execution/presto_cpp/main/TaskManager.cpp +++ b/presto-native-execution/presto_cpp/main/TaskManager.cpp @@ -680,8 +680,7 @@ TaskManager::deleteTask(const TaskId& taskId, bool /*abort*/, bool summarize) { if (state == exec::TaskState::kRunning) { execTask->requestAbort(); } - prestoTask->info.stats.endTime = - util::toISOTimestamp(velox::getCurrentTimeMs()); + prestoTask->info.stats.endTimeInMillis = velox::getCurrentTimeMs(); prestoTask->updateInfoLocked(summarize); } else { // If task is not found than we observe DELETE message coming before @@ -1077,8 +1076,7 @@ std::shared_ptr TaskManager::findOrCreateTask( prestoTask = std::make_shared(taskId, nodeId_, startProcessCpuTime); - prestoTask->info.stats.createTime = - util::toISOTimestamp(velox::getCurrentTimeMs()); + prestoTask->info.stats.createTimeInMillis = velox::getCurrentTimeMs(); prestoTask->info.needsPlan = true; prestoTask->info.metadataUpdates.connectorId = "unused"; diff --git a/presto-native-execution/presto_cpp/main/common/Utils.cpp b/presto-native-execution/presto_cpp/main/common/Utils.cpp index bc4b9ce63a925..c673f77d1fc10 100644 --- a/presto-native-execution/presto_cpp/main/common/Utils.cpp +++ b/presto-native-execution/presto_cpp/main/common/Utils.cpp @@ -19,7 +19,7 @@ namespace facebook::presto::util { -protocol::DateTime toISOTimestamp(uint64_t timeMilli) { +DateTime toISOTimestamp(uint64_t timeMilli) { char buf[80]; time_t timeSecond = timeMilli / 1000; tm gmtTime; diff --git a/presto-native-execution/presto_cpp/main/common/Utils.h b/presto-native-execution/presto_cpp/main/common/Utils.h index d481faa0af363..48d9a97d691ec 100644 --- a/presto-native-execution/presto_cpp/main/common/Utils.h +++ b/presto-native-execution/presto_cpp/main/common/Utils.h @@ -25,7 +25,8 @@ namespace facebook::presto::util { #define PRESTO_SHUTDOWN_LOG(severity) \ LOG(severity) << PRESTO_SHUTDOWN_LOG_PREFIX -protocol::DateTime toISOTimestamp(uint64_t timeMilli); +using DateTime = std::string; +DateTime toISOTimestamp(uint64_t timeMilli); std::shared_ptr createSSLContext( const std::string& clientCertAndKeyPath, diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index b3d827376c88d..f04dbeb7baaaf 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -4090,10 +4090,10 @@ void to_json(json& j, const DriverStats& p) { j = json::object(); to_json_key(j, "lifespan", p.lifespan, "DriverStats", "Lifespan", "lifespan"); to_json_key( - j, "createTime", p.createTime, "DriverStats", "DateTime", "createTime"); + j, "createTimeInMillis", p.createTimeInMillis, "DriverStats", "int64_t", "createTimeInMillis"); to_json_key( - j, "startTime", p.startTime, "DriverStats", "DateTime", "startTime"); - to_json_key(j, "endTime", p.endTime, "DriverStats", "DateTime", "endTime"); + j, "startTimeInMillis", p.startTimeInMillis, "DriverStats", "int64_t", "startTimeInMillis"); + to_json_key(j, "endTimeInMillis", p.endTimeInMillis, "DriverStats", "int64_t", "endTimeInMillis"); to_json_key( j, "queuedTime", p.queuedTime, "DriverStats", "Duration", "queuedTime"); to_json_key( @@ -4230,10 +4230,10 @@ void from_json(const json& j, DriverStats& p) { from_json_key( j, "lifespan", p.lifespan, "DriverStats", "Lifespan", "lifespan"); from_json_key( - j, "createTime", p.createTime, "DriverStats", "DateTime", "createTime"); + j, "createTimeInMillis", p.createTimeInMillis, "DriverStats", "int64_t", "createTimeInMillis"); from_json_key( - j, "startTime", p.startTime, "DriverStats", "DateTime", "startTime"); - from_json_key(j, "endTime", p.endTime, "DriverStats", "DateTime", "endTime"); + j, "startTimeInMillis", p.startTimeInMillis, "DriverStats", "int64_t", "startTimeInMillis"); + from_json_key(j, "endTimeInMillis", p.endTimeInMillis, "DriverStats", "int64_t", "endTimeInMillis"); from_json_key( j, "queuedTime", p.queuedTime, "DriverStats", "Duration", "queuedTime"); from_json_key( @@ -6973,25 +6973,25 @@ void to_json(json& j, const PipelineStats& p) { j, "pipelineId", p.pipelineId, "PipelineStats", "int", "pipelineId"); to_json_key( j, - "firstStartTime", - p.firstStartTime, + "firstStartTimeInMillis", + p.firstStartTimeInMillis, "PipelineStats", - "DateTime", - "firstStartTime"); + "int64_t", + "firstStartTimeInMillis"); to_json_key( j, - "lastStartTime", - p.lastStartTime, + "lastStartTimeInMillis", + p.lastStartTimeInMillis, "PipelineStats", - "DateTime", - "lastStartTime"); + "int64_t", + "lastStartTimeInMillis"); to_json_key( j, - "lastEndTime", - p.lastEndTime, + "lastEndTimeInMillis", + p.lastEndTimeInMillis, "PipelineStats", - "DateTime", - "lastEndTime"); + "int64_t", + "lastEndTimeInMillis"); to_json_key( j, "inputPipeline", @@ -7211,25 +7211,25 @@ void from_json(const json& j, PipelineStats& p) { j, "pipelineId", p.pipelineId, "PipelineStats", "int", "pipelineId"); from_json_key( j, - "firstStartTime", - p.firstStartTime, + "firstStartTimeInMillis", + p.firstStartTimeInMillis, "PipelineStats", - "DateTime", - "firstStartTime"); + "int64_t", + "firstStartTimeInMillis"); from_json_key( j, - "lastStartTime", - p.lastStartTime, + "lastStartTimeInMillis", + p.lastStartTimeInMillis, "PipelineStats", - "DateTime", - "lastStartTime"); + "int64_t", + "lastStartTimeInMillis"); from_json_key( j, - "lastEndTime", - p.lastEndTime, + "lastEndTimeInMillis", + p.lastEndTimeInMillis, "PipelineStats", - "DateTime", - "lastEndTime"); + "int64_t", + "lastEndTimeInMillis"); from_json_key( j, "inputPipeline", @@ -9766,24 +9766,24 @@ namespace facebook::presto::protocol { void to_json(json& j, const TaskStats& p) { j = json::object(); to_json_key( - j, "createTime", p.createTime, "TaskStats", "DateTime", "createTime"); + j, "createTimeInMillis", p.createTimeInMillis, "TaskStats", "int64_t", "createTimeInMillis"); to_json_key( j, - "firstStartTime", - p.firstStartTime, + "firstStartTimeInMillis", + p.firstStartTimeInMillis, "TaskStats", - "DateTime", - "firstStartTime"); + "int64_t", + "firstStartTimeInMillis"); to_json_key( j, - "lastStartTime", - p.lastStartTime, + "lastStartTimeInMillis", + p.lastStartTimeInMillis, "TaskStats", - "DateTime", - "lastStartTime"); + "int64_t", + "lastStartTimeInMillis"); to_json_key( - j, "lastEndTime", p.lastEndTime, "TaskStats", "DateTime", "lastEndTime"); - to_json_key(j, "endTime", p.endTime, "TaskStats", "DateTime", "endTime"); + j, "lastEndTimeInMillis", p.lastEndTimeInMillis, "TaskStats", "int64_t", "lastEndTimeInMillis"); + to_json_key(j, "endTimeInMillis", p.endTimeInMillis, "TaskStats", "int64_t", "endTimeInMillis"); to_json_key( j, "elapsedTimeInNanos", @@ -10020,24 +10020,24 @@ void to_json(json& j, const TaskStats& p) { void from_json(const json& j, TaskStats& p) { from_json_key( - j, "createTime", p.createTime, "TaskStats", "DateTime", "createTime"); + j, "createTimeInMillis", p.createTimeInMillis, "TaskStats", "int64_t", "createTimeInMillis"); from_json_key( j, - "firstStartTime", - p.firstStartTime, + "firstStartTimeInMillis", + p.firstStartTimeInMillis, "TaskStats", - "DateTime", - "firstStartTime"); + "int64_t", + "firstStartTimeInMillis"); from_json_key( j, - "lastStartTime", - p.lastStartTime, + "lastStartTimeInMillis", + p.lastStartTimeInMillis, "TaskStats", - "DateTime", - "lastStartTime"); + "int64_t", + "lastStartTimeInMillis"); from_json_key( - j, "lastEndTime", p.lastEndTime, "TaskStats", "DateTime", "lastEndTime"); - from_json_key(j, "endTime", p.endTime, "TaskStats", "DateTime", "endTime"); + j, "lastEndTimeInMillis", p.lastEndTimeInMillis, "TaskStats", "int64_t", "lastEndTimeInMillis"); + from_json_key(j, "endTimeInMillis", p.endTimeInMillis, "TaskStats", "int64_t", "endTimeInMillis"); from_json_key( j, "elapsedTimeInNanos", @@ -10577,11 +10577,11 @@ void to_json(json& j, const TaskInfo& p) { j, "taskStatus", p.taskStatus, "TaskInfo", "TaskStatus", "taskStatus"); to_json_key( j, - "lastHeartbeat", - p.lastHeartbeat, + "lastHeartbeatInMillis", + p.lastHeartbeatInMillis, "TaskInfo", - "DateTime", - "lastHeartbeat"); + "int64_t", + "lastHeartbeatInMillis"); to_json_key( j, "outputBuffers", @@ -10614,11 +10614,11 @@ void from_json(const json& j, TaskInfo& p) { j, "taskStatus", p.taskStatus, "TaskInfo", "TaskStatus", "taskStatus"); from_json_key( j, - "lastHeartbeat", - p.lastHeartbeat, + "lastHeartbeatInMillis", + p.lastHeartbeatInMillis, "TaskInfo", - "DateTime", - "lastHeartbeat"); + "int64_t", + "lastHeartbeatInMillis"); from_json_key( j, "outputBuffers", diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h index f883238cd03fe..44b104ac14b08 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h @@ -104,7 +104,6 @@ using Subfield = std::string; using HiveType = std::string; using Type = std::string; -using DateTime = std::string; using Locale = std::string; using TimeZoneKey = long; using URI = std::string; @@ -1150,9 +1149,9 @@ void from_json(const json& j, OperatorStats& p); namespace facebook::presto::protocol { struct DriverStats { Lifespan lifespan = {}; - DateTime createTime = {}; - DateTime startTime = {}; - DateTime endTime = {}; + long createTimeInMillis = {}; + long startTimeInMillis = {}; + long endTimeInMillis = {}; Duration queuedTime = {}; Duration elapsedTime = {}; int64_t userMemoryReservationInBytes = {}; @@ -1721,9 +1720,9 @@ void from_json(const json& j, PartialAggregationStatsEstimate& p); namespace facebook::presto::protocol { struct PipelineStats { int pipelineId = {}; - DateTime firstStartTime = {}; - DateTime lastStartTime = {}; - DateTime lastEndTime = {}; + int64_t firstStartTimeInMillis = {}; + int64_t lastStartTimeInMillis = {}; + int64_t lastEndTimeInMillis = {}; bool inputPipeline = {}; bool outputPipeline = {}; int totalDrivers = {}; @@ -2401,11 +2400,11 @@ void from_json(const json& j, TableWriterNode& p); } // namespace facebook::presto::protocol namespace facebook::presto::protocol { struct TaskStats { - DateTime createTime = {}; - DateTime firstStartTime = {}; - DateTime lastStartTime = {}; - DateTime lastEndTime = {}; - DateTime endTime = {}; + int64_t createTimeInMillis = {}; + int64_t firstStartTimeInMillis = {}; + int64_t lastStartTimeInMillis = {}; + int64_t lastEndTimeInMillis = {}; + int64_t endTimeInMillis = {}; int64_t elapsedTimeInNanos = {}; int64_t queuedTimeInNanos = {}; int totalDrivers = {}; @@ -2482,7 +2481,7 @@ namespace facebook::presto::protocol { struct TaskInfo { TaskId taskId = {}; TaskStatus taskStatus = {}; - DateTime lastHeartbeat = {}; + int64_t lastHeartbeatInMillis = {}; OutputBufferInfo outputBuffers = {}; List noMoreSplits = {}; TaskStats stats = {}; diff --git a/presto-spark-base/pom.xml b/presto-spark-base/pom.xml index d73651fbdb800..6a9a630cf469d 100644 --- a/presto-spark-base/pom.xml +++ b/presto-spark-base/pom.xml @@ -184,11 +184,6 @@ jol-core - - joda-time - joda-time - - javax.validation validation-api diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java index fa3b61c2631f0..666dbf699f870 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java @@ -116,7 +116,6 @@ import org.apache.spark.api.java.JavaFutureAction; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.util.CollectionAccumulator; -import org.joda.time.DateTime; import scala.Option; import javax.inject.Inject; @@ -423,7 +422,7 @@ public static StageInfo createStageInfo(QueryId queryId, SubPlan plan, ListMulti StageExecutionState.FINISHED, Optional.empty(), taskInfos, - DateTime.now(), + System.currentTimeMillis(), new Distribution().snapshot(), new RuntimeStats(), peakUserMemoryReservationInBytes, diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkTaskExecutorFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkTaskExecutorFactory.java index 71f62f3320b83..6b0cce420f68a 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkTaskExecutorFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkTaskExecutorFactory.java @@ -105,7 +105,6 @@ import io.airlift.units.DataSize; import org.apache.spark.broadcast.Broadcast; import org.apache.spark.util.CollectionAccumulator; -import org.joda.time.DateTime; import scala.Tuple2; import scala.collection.AbstractIterator; import scala.collection.Iterator; @@ -936,7 +935,7 @@ private static TaskInfo createTaskInfo( taskStats.getFullGcCount(), taskStats.getFullGcTimeInMillis(), taskStats.getTotalCpuTimeInNanos(), - System.currentTimeMillis() - taskStats.getCreateTime().getMillis(), + System.currentTimeMillis() - taskStats.getCreateTimeInMillis(), taskStats.getQueuedPartitionedSplitsWeight(), taskStats.getRunningPartitionedSplitsWeight()); @@ -954,7 +953,7 @@ private static TaskInfo createTaskInfo( return new TaskInfo( taskId, taskStatus, - DateTime.now(), + System.currentTimeMillis(), outputBufferInfo, ImmutableSet.of(), taskStats, diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/execution/http/TestPrestoSparkHttpClient.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/execution/http/TestPrestoSparkHttpClient.java index 8af8d3e929234..7e5bee83a25ef 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/execution/http/TestPrestoSparkHttpClient.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/execution/http/TestPrestoSparkHttpClient.java @@ -66,7 +66,6 @@ import io.airlift.slice.Slices; import io.airlift.units.DataSize; import io.airlift.units.Duration; -import org.joda.time.DateTime; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -1231,7 +1230,7 @@ public Response createTaskInfoResponse(HttpStatus httpStatus, String taskId) TaskId.valueOf(taskId), location, new ArrayList<>(), - new TaskStats(DateTime.now(), null), + new TaskStats(System.currentTimeMillis(), 0L), "dummy-node").withTaskStatus(createTaskStatusDone(location)); return new TestingResponse( httpStatus.code(), diff --git a/presto-tests/src/test/java/com/facebook/presto/memory/TestClusterMemoryLeakDetector.java b/presto-tests/src/test/java/com/facebook/presto/memory/TestClusterMemoryLeakDetector.java index c0601695d8e69..f1cdd711421dd 100644 --- a/presto-tests/src/test/java/com/facebook/presto/memory/TestClusterMemoryLeakDetector.java +++ b/presto-tests/src/test/java/com/facebook/presto/memory/TestClusterMemoryLeakDetector.java @@ -82,8 +82,8 @@ private static BasicQueryInfo createQueryInfo(String queryId, QueryState state) URI.create("1"), "", new BasicQueryStats( - DateTime.parse("1991-09-06T05:00-05:30"), - DateTime.parse("1991-09-06T05:01-05:30"), + DateTime.parse("1991-09-06T05:00-05:30").getMillis(), + DateTime.parse("1991-09-06T05:01-05:30").getMillis(), Duration.valueOf("4m"), Duration.valueOf("8m"), Duration.valueOf("7m"), diff --git a/presto-tests/src/test/java/com/facebook/presto/server/TestTaskInfoResource.java b/presto-tests/src/test/java/com/facebook/presto/server/TestTaskInfoResource.java index c24197ffa6e64..03ce32fbe7cf0 100644 --- a/presto-tests/src/test/java/com/facebook/presto/server/TestTaskInfoResource.java +++ b/presto-tests/src/test/java/com/facebook/presto/server/TestTaskInfoResource.java @@ -109,7 +109,7 @@ public void compareTasks(TaskInfo expectedTask, TaskInfo actualTask) assertEquals(expectedTask.getTaskId(), actualTask.getTaskId()); assertEquals(expectedTask.getTaskStatus().getState(), actualTask.getTaskStatus().getState()); assertEquals(expectedTask.getTaskStatus().getSelf(), actualTask.getTaskStatus().getSelf()); - assertEquals(expectedTask.getStats().getCreateTime(), actualTask.getStats().getCreateTime()); + assertEquals(expectedTask.getStats().getCreateTimeInMillis(), actualTask.getStats().getCreateTimeInMillis()); } public URI taskUri(String path, String taskId) diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryManager.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryManager.java index db80b24d922bd..815546f729f75 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryManager.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryManager.java @@ -352,9 +352,9 @@ private MockExecution(EventsBuilder eventsBuilder, QueryInfo info) } @Override - public DateTime getCreateTime() + public long getCreateTimeInMillis() { - return info.getQueryStats().getCreateTime(); + return info.getQueryStats().getCreateTimeInMillis(); } @Override @@ -424,10 +424,10 @@ private static QueryInfo mockInfo(String query, String queryId, QueryState state Optional.empty(), Optional.empty(), new QueryStats( - DateTime.parse("1991-09-06T05:00-05:30"), - DateTime.parse("1991-09-06T05:01-05:30"), - DateTime.parse("1991-09-06T05:02-05:30"), - DateTime.parse("1991-09-06T06:00-05:30"), + DateTime.parse("1991-09-06T05:00-05:30").getMillis(), + DateTime.parse("1991-09-06T05:01-05:30").getMillis(), + DateTime.parse("1991-09-06T05:02-05:30").getMillis(), + DateTime.parse("1991-09-06T06:00-05:30").getMillis(), Duration.valueOf("8m"), Duration.valueOf("5m"), Duration.valueOf("7m"), diff --git a/presto-ui/src/components/QueryDetail.jsx b/presto-ui/src/components/QueryDetail.jsx index 15c77e1d6d26f..31db7f39fe4a9 100644 --- a/presto-ui/src/components/QueryDetail.jsx +++ b/presto-ui/src/components/QueryDetail.jsx @@ -1381,7 +1381,7 @@ export class QueryDetail extends React.Component { Completion Time - {query.queryStats.endTime ? formatShortDateTime(new Date(query.queryStats.endTime)) : ""} + {new Date(query.queryStats.endTime).getTime() !== 0 ? formatShortDateTime(new Date(query.queryStats.endTime)) : ""} diff --git a/presto-ui/src/components/QueryOverview.jsx b/presto-ui/src/components/QueryOverview.jsx index 1acd1b895326f..77d07f15635d9 100644 --- a/presto-ui/src/components/QueryOverview.jsx +++ b/presto-ui/src/components/QueryOverview.jsx @@ -1338,7 +1338,7 @@ export default function QueryOverview({ data, show }: { data: QueryData, show: b Completion Time - {data.queryStats.endTime ? formatShortDateTime(new Date(data.queryStats.endTime)) : ""} + {new Date(data.queryStats.endTime).getTime() !== 0 ? formatShortDateTime(new Date(data.queryStats.endTime)) : ""}