diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java index 383674e725116..1c1d98580f500 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java @@ -19,8 +19,6 @@ import com.facebook.presto.client.FailureInfo; import com.facebook.presto.client.QueryError; import com.facebook.presto.client.QueryResults; -import com.facebook.presto.client.StageStats; -import com.facebook.presto.client.StatementStats; import com.facebook.presto.common.Page; import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.BooleanType; @@ -30,11 +28,7 @@ import com.facebook.presto.execution.QueryInfo; import com.facebook.presto.execution.QueryManager; import com.facebook.presto.execution.QueryState; -import com.facebook.presto.execution.QueryStats; -import com.facebook.presto.execution.StageExecutionInfo; -import com.facebook.presto.execution.StageExecutionStats; import com.facebook.presto.execution.StageInfo; -import com.facebook.presto.execution.TaskInfo; import com.facebook.presto.execution.buffer.PagesSerdeFactory; import com.facebook.presto.operator.ExchangeClient; import com.facebook.presto.spi.ErrorCode; @@ -61,7 +55,6 @@ import javax.ws.rs.core.UriInfo; import java.net.URI; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -76,6 +69,7 @@ import static com.facebook.presto.SystemSessionProperties.getTargetResultSize; import static com.facebook.presto.SystemSessionProperties.isExchangeCompressionEnabled; import static com.facebook.presto.execution.QueryState.FAILED; +import static com.facebook.presto.server.protocol.QueryResourceUtil.toStatementStats; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.util.Failures.toFailure; import static com.google.common.base.MoreObjects.firstNonNull; @@ -523,90 +517,6 @@ private synchronized URI createNextResultsUri(String scheme, UriInfo uriInfo, lo return uri.build(); } - private static StatementStats toStatementStats(QueryInfo queryInfo) - { - QueryStats queryStats = queryInfo.getQueryStats(); - StageInfo outputStage = queryInfo.getOutputStage().orElse(null); - - return StatementStats.builder() - .setState(queryInfo.getState().toString()) - .setQueued(queryInfo.getState() == QueryState.QUEUED) - .setScheduled(queryInfo.isScheduled()) - .setNodes(globalUniqueNodes(outputStage).size()) - .setTotalSplits(queryStats.getTotalDrivers()) - .setQueuedSplits(queryStats.getQueuedDrivers()) - .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) - .setCompletedSplits(queryStats.getCompletedDrivers()) - .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) - .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) - .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) - .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) - .setProcessedRows(queryStats.getRawInputPositions()) - .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) - .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) - .setPeakTotalMemoryBytes(queryStats.getPeakTotalMemoryReservation().toBytes()) - .setPeakTaskTotalMemoryBytes(queryStats.getPeakTaskTotalMemory().toBytes()) - .setSpilledBytes(queryStats.getSpilledDataSize().toBytes()) - .setRootStage(toStageStats(outputStage)) - .build(); - } - - private static StageStats toStageStats(StageInfo stageInfo) - { - if (stageInfo == null) { - return null; - } - - StageExecutionInfo currentStageExecutionInfo = stageInfo.getLatestAttemptExecutionInfo(); - StageExecutionStats stageExecutionStats = currentStageExecutionInfo.getStats(); - - ImmutableList.Builder subStages = ImmutableList.builder(); - for (StageInfo subStage : stageInfo.getSubStages()) { - subStages.add(toStageStats(subStage)); - } - - Set uniqueNodes = new HashSet<>(); - for (TaskInfo task : currentStageExecutionInfo.getTasks()) { - // todo add nodeId to TaskInfo - URI uri = task.getTaskStatus().getSelf(); - uniqueNodes.add(uri.getHost() + ":" + uri.getPort()); - } - - return StageStats.builder() - .setStageId(String.valueOf(stageInfo.getStageId().getId())) - .setState(currentStageExecutionInfo.getState().toString()) - .setDone(currentStageExecutionInfo.getState().isDone()) - .setNodes(uniqueNodes.size()) - .setTotalSplits(stageExecutionStats.getTotalDrivers()) - .setQueuedSplits(stageExecutionStats.getQueuedDrivers()) - .setRunningSplits(stageExecutionStats.getRunningDrivers() + stageExecutionStats.getBlockedDrivers()) - .setCompletedSplits(stageExecutionStats.getCompletedDrivers()) - .setCpuTimeMillis(stageExecutionStats.getTotalCpuTime().toMillis()) - .setWallTimeMillis(stageExecutionStats.getTotalScheduledTime().toMillis()) - .setProcessedRows(stageExecutionStats.getRawInputPositions()) - .setProcessedBytes(stageExecutionStats.getRawInputDataSize().toBytes()) - .setSubStages(subStages.build()) - .build(); - } - - private static Set globalUniqueNodes(StageInfo stageInfo) - { - if (stageInfo == null) { - return ImmutableSet.of(); - } - ImmutableSet.Builder nodes = ImmutableSet.builder(); - for (TaskInfo task : stageInfo.getLatestAttemptExecutionInfo().getTasks()) { - // todo add nodeId to TaskInfo - URI uri = task.getTaskStatus().getSelf(); - nodes.add(uri.getHost() + ":" + uri.getPort()); - } - - for (StageInfo subStage : stageInfo.getSubStages()) { - nodes.addAll(globalUniqueNodes(subStage)); - } - return nodes.build(); - } - private static URI findCancelableLeafStage(QueryInfo queryInfo) { // if query is running, find the leaf-most running stage diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/QueryResourceUtil.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/QueryResourceUtil.java index 66d8dda725624..bd981354d8876 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/QueryResourceUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/QueryResourceUtil.java @@ -14,12 +14,26 @@ package com.facebook.presto.server.protocol; import com.facebook.presto.client.QueryResults; +import com.facebook.presto.client.StageStats; +import com.facebook.presto.client.StatementStats; +import com.facebook.presto.execution.QueryInfo; +import com.facebook.presto.execution.QueryState; +import com.facebook.presto.execution.QueryStats; +import com.facebook.presto.execution.StageExecutionInfo; +import com.facebook.presto.execution.StageExecutionStats; +import com.facebook.presto.execution.StageInfo; +import com.facebook.presto.execution.TaskInfo; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import javax.ws.rs.core.Response; import java.io.UnsupportedEncodingException; +import java.net.URI; import java.net.URLEncoder; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import static com.facebook.presto.client.PrestoHeaders.PRESTO_ADDED_PREPARE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLEAR_SESSION; @@ -79,6 +93,34 @@ public static Response toResponse(Query query, QueryResults queryResults) return response.build(); } + public static StatementStats toStatementStats(QueryInfo queryInfo) + { + QueryStats queryStats = queryInfo.getQueryStats(); + StageInfo outputStage = queryInfo.getOutputStage().orElse(null); + + return StatementStats.builder() + .setState(queryInfo.getState().toString()) + .setQueued(queryInfo.getState() == QueryState.QUEUED) + .setScheduled(queryInfo.isScheduled()) + .setNodes(globalUniqueNodes(outputStage).size()) + .setTotalSplits(queryStats.getTotalDrivers()) + .setQueuedSplits(queryStats.getQueuedDrivers()) + .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) + .setCompletedSplits(queryStats.getCompletedDrivers()) + .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) + .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) + .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) + .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) + .setProcessedRows(queryStats.getRawInputPositions()) + .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) + .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) + .setPeakTotalMemoryBytes(queryStats.getPeakTotalMemoryReservation().toBytes()) + .setPeakTaskTotalMemoryBytes(queryStats.getPeakTaskTotalMemory().toBytes()) + .setSpilledBytes(queryStats.getSpilledDataSize().toBytes()) + .setRootStage(toStageStats(outputStage)) + .build(); + } + private static String urlEncode(String value) { try { @@ -88,4 +130,60 @@ private static String urlEncode(String value) throw new AssertionError(e); } } + + private static Set globalUniqueNodes(StageInfo stageInfo) + { + if (stageInfo == null) { + return ImmutableSet.of(); + } + ImmutableSet.Builder nodes = ImmutableSet.builder(); + for (TaskInfo task : stageInfo.getLatestAttemptExecutionInfo().getTasks()) { + // todo add nodeId to TaskInfo + URI uri = task.getTaskStatus().getSelf(); + nodes.add(uri.getHost() + ":" + uri.getPort()); + } + + for (StageInfo subStage : stageInfo.getSubStages()) { + nodes.addAll(globalUniqueNodes(subStage)); + } + return nodes.build(); + } + + private static StageStats toStageStats(StageInfo stageInfo) + { + if (stageInfo == null) { + return null; + } + + StageExecutionInfo currentStageExecutionInfo = stageInfo.getLatestAttemptExecutionInfo(); + StageExecutionStats stageExecutionStats = currentStageExecutionInfo.getStats(); + + ImmutableList.Builder subStages = ImmutableList.builder(); + for (StageInfo subStage : stageInfo.getSubStages()) { + subStages.add(toStageStats(subStage)); + } + + Set uniqueNodes = new HashSet<>(); + for (TaskInfo task : currentStageExecutionInfo.getTasks()) { + // todo add nodeId to TaskInfo + URI uri = task.getTaskStatus().getSelf(); + uniqueNodes.add(uri.getHost() + ":" + uri.getPort()); + } + + return StageStats.builder() + .setStageId(String.valueOf(stageInfo.getStageId().getId())) + .setState(currentStageExecutionInfo.getState().toString()) + .setDone(currentStageExecutionInfo.getState().isDone()) + .setNodes(uniqueNodes.size()) + .setTotalSplits(stageExecutionStats.getTotalDrivers()) + .setQueuedSplits(stageExecutionStats.getQueuedDrivers()) + .setRunningSplits(stageExecutionStats.getRunningDrivers() + stageExecutionStats.getBlockedDrivers()) + .setCompletedSplits(stageExecutionStats.getCompletedDrivers()) + .setCpuTimeMillis(stageExecutionStats.getTotalCpuTime().toMillis()) + .setWallTimeMillis(stageExecutionStats.getTotalScheduledTime().toMillis()) + .setProcessedRows(stageExecutionStats.getRawInputPositions()) + .setProcessedBytes(stageExecutionStats.getRawInputDataSize().toBytes()) + .setSubStages(subStages.build()) + .build(); + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/PrestoVerifyCommand.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/PrestoVerifyCommand.java index 64237d418a09a..c4b17512087aa 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/PrestoVerifyCommand.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/PrestoVerifyCommand.java @@ -55,6 +55,12 @@ public Set getCustomEventClientTypes() return ImmutableSet.of(); } + @Override + public Set getCustomQueryActionTypes() + { + return ImmutableSet.of(); + } + @Override public List>> getCustomQueryFilterClasses() { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForHelper.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForHelper.java new file mode 100644 index 0000000000000..3390f762fcd9f --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForHelper.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.annotation; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface ForHelper +{ +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java index 781057ff08d8d..a56c337cf752a 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java @@ -43,13 +43,14 @@ public class QueryInfo private final Double wallTimeSecs; private final Long peakTotalMemoryBytes; private final Long peakTaskTotalMemoryBytes; + private final QueryStatsEvent queryStats; public QueryInfo( String catalog, String schema, String originalQuery) { - this(catalog, schema, originalQuery, Optional.empty(), ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + this(catalog, schema, originalQuery, Optional.empty(), ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } public QueryInfo( @@ -67,7 +68,8 @@ public QueryInfo( Optional cpuTimeSecs, Optional wallTimeSecs, Optional peakTotalMemoryBytes, - Optional peakTaskTotalMemoryBytes) + Optional peakTaskTotalMemoryBytes, + Optional queryStats) { this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); @@ -84,6 +86,7 @@ public QueryInfo( this.wallTimeSecs = wallTimeSecs.orElse(null); this.peakTotalMemoryBytes = peakTotalMemoryBytes.orElse(null); this.peakTaskTotalMemoryBytes = peakTaskTotalMemoryBytes.orElse(null); + this.queryStats = queryStats.orElse(null); } @EventField @@ -175,4 +178,10 @@ public Long getPeakTaskTotalMemoryBytes() { return peakTaskTotalMemoryBytes; } + + @EventField + public QueryStatsEvent getQueryStats() + { + return queryStats; + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryStatsEvent.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryStatsEvent.java new file mode 100644 index 0000000000000..bee1ce9006598 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryStatsEvent.java @@ -0,0 +1,229 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.event; + +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventType; +import com.facebook.presto.jdbc.QueryStats; + +import javax.annotation.concurrent.Immutable; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +@Immutable +@EventType("QueryStatsEvent") +public class QueryStatsEvent +{ + private final String queryId; + private final String state; + private final boolean queued; + private final boolean scheduled; + private final int nodes; + private final int totalSplits; + private final int queuedSplits; + private final int runningSplits; + private final int completedSplits; + private final long cpuTimeMillis; + private final long wallTimeMillis; + private final long queuedTimeMillis; + private final long elapsedTimeMillis; + private final long processedRows; + private final long processedBytes; + private final long peakMemoryBytes; + private final long peakTotalMemoryBytes; + private final long peakTaskTotalMemoryBytes; + private final String extraData; + + public QueryStatsEvent(QueryStats stats, Optional extraData) + { + this( + stats.getQueryId(), + stats.getState(), + stats.isQueued(), + stats.isScheduled(), + stats.getNodes(), + stats.getTotalSplits(), + stats.getQueuedSplits(), + stats.getRunningSplits(), + stats.getCompletedSplits(), + stats.getCpuTimeMillis(), + stats.getWallTimeMillis(), + stats.getQueuedTimeMillis(), + stats.getElapsedTimeMillis(), + stats.getProcessedRows(), + stats.getProcessedBytes(), + stats.getPeakMemoryBytes(), + stats.getPeakTotalMemoryBytes(), + stats.getPeakTaskTotalMemoryBytes(), + extraData); + } + + public QueryStatsEvent( + String queryId, + String state, + boolean queued, + boolean scheduled, + int nodes, + int totalSplits, + int queuedSplits, + int runningSplits, + int completedSplits, + long cpuTimeMillis, + long wallTimeMillis, + long queuedTimeMillis, + long elapsedTimeMillis, + long processedRows, + long processedBytes, + long peakMemoryBytes, + long peakTotalMemoryBytes, + long peakTaskTotalMemoryBytes, + Optional extraData) + { + this.queryId = requireNonNull(queryId, "queryId is null"); + this.state = requireNonNull(state, "state is null"); + this.queued = queued; + this.scheduled = scheduled; + this.nodes = nodes; + this.totalSplits = totalSplits; + this.queuedSplits = queuedSplits; + this.runningSplits = runningSplits; + this.completedSplits = completedSplits; + this.cpuTimeMillis = cpuTimeMillis; + this.wallTimeMillis = wallTimeMillis; + this.queuedTimeMillis = queuedTimeMillis; + this.elapsedTimeMillis = elapsedTimeMillis; + this.processedRows = processedRows; + this.processedBytes = processedBytes; + this.peakMemoryBytes = peakMemoryBytes; + this.peakTotalMemoryBytes = peakTotalMemoryBytes; + this.peakTaskTotalMemoryBytes = peakTaskTotalMemoryBytes; + this.extraData = extraData.orElse(null); + } + + @EventField + public String getQueryId() + { + return queryId; + } + + @EventField + public String getState() + { + return state; + } + + @EventField + public boolean isQueued() + { + return queued; + } + + @EventField + public boolean isScheduled() + { + return scheduled; + } + + @EventField + public int getNodes() + { + return nodes; + } + + @EventField + public int getTotalSplits() + { + return totalSplits; + } + + @EventField + public int getQueuedSplits() + { + return queuedSplits; + } + + @EventField + public int getRunningSplits() + { + return runningSplits; + } + + @EventField + public int getCompletedSplits() + { + return completedSplits; + } + + @EventField + public long getCpuTimeMillis() + { + return cpuTimeMillis; + } + + @EventField + public long getWallTimeMillis() + { + return wallTimeMillis; + } + + @EventField + public long getQueuedTimeMillis() + { + return queuedTimeMillis; + } + + @EventField + public long getElapsedTimeMillis() + { + return elapsedTimeMillis; + } + + @EventField + public long getProcessedRows() + { + return processedRows; + } + + @EventField + public long getProcessedBytes() + { + return processedBytes; + } + + @EventField + public long getPeakMemoryBytes() + { + return peakMemoryBytes; + } + + @EventField + public long getPeakTotalMemoryBytes() + { + return peakTotalMemoryBytes; + } + + @EventField + public long getPeakTaskTotalMemoryBytes() + { + return peakTaskTotalMemoryBytes; + } + + @EventField + public String getExtraData() + { + return extraData; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java index e3866c7c14ec1..d8a34f2895ed1 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java @@ -13,15 +13,17 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.SqlFormatter; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.verifier.event.DeterminismAnalysisDetails; import com.facebook.presto.verifier.event.QueryInfo; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; import com.facebook.presto.verifier.framework.MatchResult.MatchType; import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.prestoaction.QueryAction; +import com.facebook.presto.verifier.prestoaction.QueryActions; import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; import com.facebook.presto.verifier.resolver.FailureResolverManager; import com.facebook.presto.verifier.rewrite.QueryRewriter; @@ -67,7 +69,7 @@ public abstract class AbstractVerification { private static final String INTERNAL_ERROR = "VERIFIER_INTERNAL_ERROR"; - private final PrestoAction prestoAction; + private final QueryActions queryActions; private final SourceQuery sourceQuery; private final QueryRewriter queryRewriter; private final DeterminismAnalyzer determinismAnalyzer; @@ -79,8 +81,11 @@ public abstract class AbstractVerification private final boolean smartTeardown; private final int verificationResubmissionLimit; + private final boolean setupOnMainClusters; + private final boolean teardownOnMainClusters; + public AbstractVerification( - PrestoAction prestoAction, + QueryActions queryActions, SourceQuery sourceQuery, QueryRewriter queryRewriter, DeterminismAnalyzer determinismAnalyzer, @@ -89,7 +94,7 @@ public AbstractVerification( VerificationContext verificationContext, VerifierConfig verifierConfig) { - this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); + this.queryActions = requireNonNull(queryActions, "queryActions is null"); this.sourceQuery = requireNonNull(sourceQuery, "sourceQuery is null"); this.queryRewriter = requireNonNull(queryRewriter, "queryRewriter is null"); this.determinismAnalyzer = requireNonNull(determinismAnalyzer, "determinismAnalyzer is null"); @@ -100,13 +105,15 @@ public AbstractVerification( this.testId = requireNonNull(verifierConfig.getTestId(), "testId is null"); this.smartTeardown = verifierConfig.isSmartTeardown(); this.verificationResubmissionLimit = verifierConfig.getVerificationResubmissionLimit(); + this.setupOnMainClusters = verifierConfig.isSetupOnMainClusters(); + this.teardownOnMainClusters = verifierConfig.isTeardownOnMainClusters(); } protected abstract MatchResult verify(QueryBundle control, QueryBundle test, ChecksumQueryContext controlContext, ChecksumQueryContext testContext); - protected PrestoAction getPrestoAction() + protected PrestoAction getHelperAction() { - return prestoAction; + return queryActions.getHelperAction(); } @Override @@ -146,22 +153,24 @@ public VerificationResult run() QueryBundle controlQueryBundle = control.get(); QueryBundle testQueryBundle = test.get(); + QueryAction controlSetupAction = setupOnMainClusters ? queryActions.getControlAction() : queryActions.getHelperAction(); + QueryAction testSetupAction = setupOnMainClusters ? queryActions.getTestAction() : queryActions.getHelperAction(); controlQueryBundle.getSetupQueries().forEach(query -> runAndConsume( - () -> prestoAction.execute(query, CONTROL_SETUP), + () -> controlSetupAction.execute(query, CONTROL_SETUP), controlQueryContext::addSetupQuery, controlQueryContext::setException)); runAndConsume( - () -> prestoAction.execute(controlQueryBundle.getQuery(), CONTROL_MAIN), + () -> queryActions.getControlAction().execute(controlQueryBundle.getQuery(), CONTROL_MAIN), controlQueryContext::setMainQueryStats, controlQueryContext::setException); controlQueryContext.setState(QueryState.SUCCEEDED); testQueryBundle.getSetupQueries().forEach(query -> runAndConsume( - () -> prestoAction.execute(query, TEST_SETUP), + () -> testSetupAction.execute(query, TEST_SETUP), testQueryContext::addSetupQuery, testQueryContext::setException)); runAndConsume( - () -> prestoAction.execute(testQueryBundle.getQuery(), TEST_MAIN), + () -> queryActions.getTestAction().execute(testQueryBundle.getQuery(), TEST_MAIN), testQueryContext::setMainQueryStats, testQueryContext::setException); testQueryContext.setState(QueryState.SUCCEEDED); @@ -188,8 +197,10 @@ public VerificationResult run() if (!smartTeardown || testQueryContext.getState() != QueryState.SUCCEEDED || (partialResult.isPresent() && partialResult.get().getStatus().equals(SUCCEEDED))) { - teardownSafely(prestoAction, control, controlQueryContext::addTeardownQuery); - teardownSafely(prestoAction, test, testQueryContext::addTeardownQuery); + QueryAction controlTeardownAction = teardownOnMainClusters ? queryActions.getControlAction() : queryActions.getHelperAction(); + QueryAction testTeardownAction = teardownOnMainClusters ? queryActions.getTestAction() : queryActions.getHelperAction(); + teardownSafely(controlTeardownAction, control, controlQueryContext::addTeardownQuery); + teardownSafely(testTeardownAction, test, testQueryContext::addTeardownQuery); } } @@ -323,7 +334,7 @@ private static QueryInfo buildQueryInfo( configuration.getCatalog(), configuration.getSchema(), originalQuery, - queryContext.getMainQueryStats().map(QueryStats::getQueryId), + queryContext.getMainQueryStats().map(QueryStatsEvent::getQueryId), queryContext.getSetupQueryIds(), queryContext.getTeardownQueryIds(), checksumQueryContext.getChecksumQueryId(), @@ -331,10 +342,11 @@ private static QueryInfo buildQueryInfo( queryBundle.map(QueryBundle::getSetupQueries).map(AbstractVerification::formatSqls), queryBundle.map(QueryBundle::getTeardownQueries).map(AbstractVerification::formatSqls), checksumQueryContext.getChecksumQuery(), - millisToSeconds(queryContext.getMainQueryStats().map(QueryStats::getCpuTimeMillis)), - millisToSeconds(queryContext.getMainQueryStats().map(QueryStats::getWallTimeMillis)), - queryContext.getMainQueryStats().map(QueryStats::getPeakTotalMemoryBytes), - queryContext.getMainQueryStats().map(QueryStats::getPeakTaskTotalMemoryBytes)); + millisToSeconds(queryContext.getMainQueryStats().map(QueryStatsEvent::getCpuTimeMillis)), + millisToSeconds(queryContext.getMainQueryStats().map(QueryStatsEvent::getWallTimeMillis)), + queryContext.getMainQueryStats().map(QueryStatsEvent::getPeakTotalMemoryBytes), + queryContext.getMainQueryStats().map(QueryStatsEvent::getPeakTaskTotalMemoryBytes), + queryContext.getMainQueryStats()); } protected static String formatSql(Statement statement) @@ -418,17 +430,17 @@ private String constructErrorMessage( private static class QueryContext { - private Optional mainQueryStats = Optional.empty(); + private Optional mainQueryStats = Optional.empty(); private Optional state = Optional.empty(); private ImmutableList.Builder setupQueryIds = ImmutableList.builder(); private ImmutableList.Builder teardownQueryIds = ImmutableList.builder(); - public Optional getMainQueryStats() + public Optional getMainQueryStats() { return mainQueryStats; } - public void setMainQueryStats(QueryStats mainQueryStats) + public void setMainQueryStats(QueryStatsEvent mainQueryStats) { checkState(!this.mainQueryStats.isPresent(), "mainQueryStats is already set", mainQueryStats); this.mainQueryStats = Optional.of(mainQueryStats); @@ -455,7 +467,7 @@ public List getSetupQueryIds() return setupQueryIds.build(); } - public void addSetupQuery(QueryStats queryStats) + public void addSetupQuery(QueryStatsEvent queryStats) { setupQueryIds.add(queryStats.getQueryId()); } @@ -465,7 +477,7 @@ public List getTeardownQueryIds() return teardownQueryIds.build(); } - public void addTeardownQuery(QueryStats queryStats) + public void addTeardownQuery(QueryStatsEvent queryStats) { teardownQueryIds.add(queryStats.getQueryId()); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerifyCommand.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerifyCommand.java index bc2214e22f94a..b6591cfe8213a 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerifyCommand.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerifyCommand.java @@ -17,6 +17,7 @@ import com.facebook.airlift.bootstrap.LifeCycleManager; import com.facebook.airlift.log.Logger; import com.facebook.presto.verifier.event.EventClientModule; +import com.facebook.presto.verifier.prestoaction.QueryActionsModule; import com.facebook.presto.verifier.source.SourceQueryModule; import com.google.common.collect.ImmutableList; import com.google.inject.Injector; @@ -43,10 +44,10 @@ public void run() Bootstrap app = new Bootstrap(ImmutableList.builder() .add(new VerifierModule( getSqlParserOptions(), - getCustomQueryFilterClasses(), - getSqlExceptionClassifier())) + getCustomQueryFilterClasses())) .add(new SourceQueryModule(getCustomSourceQuerySupplierTypes())) .add(new EventClientModule(getCustomEventClientTypes())) + .add(new QueryActionsModule(getSqlExceptionClassifier(), getCustomQueryActionTypes())) .addAll(getAdditionalModules()) .build()); Injector injector = null; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java index 134df2d934a7a..95ac417950310 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java @@ -17,7 +17,7 @@ import com.facebook.presto.sql.tree.Query; import com.facebook.presto.verifier.checksum.ChecksumResult; import com.facebook.presto.verifier.checksum.ChecksumValidator; -import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.prestoaction.QueryActions; import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; import com.facebook.presto.verifier.resolver.FailureResolverManager; import com.facebook.presto.verifier.rewrite.QueryRewriter; @@ -39,7 +39,7 @@ public class DataVerification private final ChecksumValidator checksumValidator; public DataVerification( - PrestoAction prestoAction, + QueryActions queryActions, SourceQuery sourceQuery, QueryRewriter queryRewriter, DeterminismAnalyzer determinismAnalyzer, @@ -50,7 +50,7 @@ public DataVerification( TypeManager typeManager, ChecksumValidator checksumValidator) { - super(prestoAction, sourceQuery, queryRewriter, determinismAnalyzer, failureResolverManager, exceptionClassifier, verificationContext, verifierConfig); + super(queryActions, sourceQuery, queryRewriter, determinismAnalyzer, failureResolverManager, exceptionClassifier, verificationContext, verifierConfig); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.checksumValidator = requireNonNull(checksumValidator, "checksumValidator is null"); } @@ -58,8 +58,8 @@ public DataVerification( @Override public MatchResult verify(QueryBundle control, QueryBundle test, ChecksumQueryContext controlContext, ChecksumQueryContext testContext) { - List controlColumns = getColumns(getPrestoAction(), typeManager, control.getTableName()); - List testColumns = getColumns(getPrestoAction(), typeManager, test.getTableName()); + List controlColumns = getColumns(getHelperAction(), typeManager, control.getTableName()); + List testColumns = getColumns(getHelperAction(), typeManager, test.getTableName()); Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getTableName(), controlColumns); Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getTableName(), testColumns); @@ -68,10 +68,10 @@ public MatchResult verify(QueryBundle control, QueryBundle test, ChecksumQueryCo testContext.setChecksumQuery(formatSql(testChecksumQuery)); QueryResult controlChecksum = callAndConsume( - () -> getPrestoAction().execute(controlChecksumQuery, CONTROL_CHECKSUM, ChecksumResult::fromResultSet), + () -> getHelperAction().execute(controlChecksumQuery, CONTROL_CHECKSUM, ChecksumResult::fromResultSet), stats -> controlContext.setChecksumQueryId(stats.getQueryId())); QueryResult testChecksum = callAndConsume( - () -> getPrestoAction().execute(testChecksumQuery, TEST_CHECKSUM, ChecksumResult::fromResultSet), + () -> getHelperAction().execute(testChecksumQuery, TEST_CHECKSUM, ChecksumResult::fromResultSet), stats -> testContext.setChecksumQueryId(stats.getQueryId())); return match(checksumValidator, controlColumns, testColumns, getOnlyElement(controlChecksum.getResults()), getOnlyElement(testChecksum.getResults())); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerificationUtil.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerificationUtil.java index 5ff5303d06fcc..5509f92ef0537 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerificationUtil.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerificationUtil.java @@ -15,14 +15,15 @@ import com.facebook.airlift.log.Logger; import com.facebook.presto.common.type.TypeManager; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.ShowColumns; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.verifier.checksum.ChecksumResult; import com.facebook.presto.verifier.checksum.ChecksumValidator; import com.facebook.presto.verifier.checksum.ColumnMatchResult; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.prestoaction.QueryAction; import com.google.common.collect.ImmutableList; import java.util.List; @@ -46,7 +47,7 @@ public class DataVerificationUtil private DataVerificationUtil() {} - public static void teardownSafely(PrestoAction prestoAction, Optional bundle, Consumer queryStatsConsumer) + public static void teardownSafely(QueryAction queryAction, Optional bundle, Consumer queryStatsConsumer) { if (!bundle.isPresent()) { return; @@ -54,7 +55,7 @@ public static void teardownSafely(PrestoAction prestoAction, Optional prestoAction.execute(teardownQuery, forTeardown(bundle.get().getCluster())), queryStatsConsumer); + runAndConsume(() -> queryAction.execute(teardownQuery, forTeardown(bundle.get().getCluster())), queryStatsConsumer); } catch (Throwable t) { log.warn("Failed to teardown %s: %s", bundle.get().getCluster().name().toLowerCase(ENGLISH), formatSql(teardownQuery, Optional.empty())); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java index f461766592404..550ec632dae1d 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java @@ -64,12 +64,11 @@ public class DeterminismAnalyzer private final QueryRewriter queryRewriter; private final ChecksumValidator checksumValidator; private final TypeManager typeManager; - private final VerificationContext verificationContext; - private boolean runTeardown; - private int maxAnalysisRuns; - private Set nonDeterministicCatalogs; - private boolean handleLimitQuery; + private final boolean runTeardown; + private final int maxAnalysisRuns; + private final Set nonDeterministicCatalogs; + private final boolean handleLimitQuery; public DeterminismAnalyzer( SourceQuery sourceQuery, @@ -77,7 +76,6 @@ public DeterminismAnalyzer( QueryRewriter queryRewriter, ChecksumValidator checksumValidator, TypeManager typeManager, - VerificationContext verificationContext, DeterminismAnalyzerConfig config) { this.sourceQuery = requireNonNull(sourceQuery, "sourceQuery is null"); @@ -85,7 +83,6 @@ public DeterminismAnalyzer( this.queryRewriter = requireNonNull(queryRewriter, "queryRewriter is null"); this.checksumValidator = requireNonNull(checksumValidator, "checksumValidator is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); - this.verificationContext = requireNonNull(verificationContext, "verificationContext is null"); this.runTeardown = config.isRunTeardown(); this.maxAnalysisRuns = config.getMaxAnalysisRuns(); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoQueryException.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoQueryException.java index fc523d167eae3..a3a87ed49ab0a 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoQueryException.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoQueryException.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCode; import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.verifier.event.QueryStatsEvent; import java.util.Optional; @@ -26,14 +26,14 @@ public class PrestoQueryException extends QueryException { private final Optional errorCode; - private final Optional queryStats; + private final Optional queryStats; public PrestoQueryException( Throwable cause, boolean retryable, QueryStage queryStage, Optional errorCode, - Optional queryStats) + Optional queryStats) { super(cause, retryable, queryStage); this.errorCode = requireNonNull(errorCode, "errorCode is null"); @@ -45,7 +45,7 @@ public Optional getErrorCode() return errorCode; } - public Optional getQueryStats() + public Optional getQueryStats() { return queryStats; } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryException.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryException.java index 2714afc163fc3..69e1ff9354aef 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryException.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryException.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCode; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.verifier.event.QueryFailure; +import com.facebook.presto.verifier.event.QueryStatsEvent; import java.util.Optional; @@ -29,6 +29,13 @@ public abstract class QueryException private final boolean retryable; private final QueryStage queryStage; + public QueryException(String message, boolean retryable, QueryStage queryStage) + { + super(message); + this.retryable = retryable; + this.queryStage = requireNonNull(queryStage, "queryStage is null"); + } + public QueryException(Throwable cause, boolean retryable, QueryStage queryStage) { super(cause); @@ -58,7 +65,7 @@ public QueryFailure toQueryFailure() : Optional.empty(), retryable, this instanceof PrestoQueryException - ? ((PrestoQueryException) this).getQueryStats().map(QueryStats::getQueryId) + ? ((PrestoQueryException) this).getQueryStats().map(QueryStatsEvent::getQueryId) : Optional.empty(), getStackTraceAsString(this)); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryResult.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryResult.java index 0055523987eda..f7b04ab5bcbb3 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryResult.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryResult.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.google.common.collect.ImmutableList; import java.sql.ResultSetMetaData; @@ -25,9 +25,9 @@ public class QueryResult { private final List results; private final ResultSetMetaData metadata; - private final QueryStats queryStats; + private final QueryStatsEvent queryStats; - public QueryResult(List results, ResultSetMetaData metadata, QueryStats queryStats) + public QueryResult(List results, ResultSetMetaData metadata, QueryStatsEvent queryStats) { this.results = ImmutableList.copyOf(results); this.metadata = requireNonNull(metadata, "metadata is null"); @@ -44,7 +44,7 @@ public ResultSetMetaData getMetadata() return metadata; } - public QueryStats getQueryStats() + public QueryStatsEvent getQueryStats() { return queryStats; } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java index 318f938a1a968..286b1445eb8b3 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java @@ -15,11 +15,9 @@ import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.verifier.annotation.ForTest; import com.facebook.presto.verifier.checksum.ChecksumValidator; -import com.facebook.presto.verifier.prestoaction.NodeResourceClient; -import com.facebook.presto.verifier.prestoaction.PrestoAction; -import com.facebook.presto.verifier.prestoaction.PrestoActionFactory; +import com.facebook.presto.verifier.prestoaction.QueryActions; +import com.facebook.presto.verifier.prestoaction.QueryActionsFactory; import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; import com.facebook.presto.verifier.resolver.FailureResolverFactoryContext; import com.facebook.presto.verifier.resolver.FailureResolverManager; @@ -38,10 +36,9 @@ public class VerificationFactory { private final SqlParser sqlParser; - private final PrestoActionFactory prestoActionFactory; + private final QueryActionsFactory queryActionsFactory; private final QueryRewriterFactory queryRewriterFactory; private final FailureResolverManagerFactory failureResolverManagerFactory; - private final NodeResourceClient testResourceClient; private final ChecksumValidator checksumValidator; private final SqlExceptionClassifier exceptionClassifier; private final VerifierConfig verifierConfig; @@ -51,10 +48,9 @@ public class VerificationFactory @Inject public VerificationFactory( SqlParser sqlParser, - PrestoActionFactory prestoActionFactory, + QueryActionsFactory queryActionsFactory, QueryRewriterFactory queryRewriterFactory, FailureResolverManagerFactory failureResolverManagerFactory, - @ForTest NodeResourceClient testResourceClient, ChecksumValidator checksumValidator, SqlExceptionClassifier exceptionClassifier, VerifierConfig verifierConfig, @@ -62,10 +58,9 @@ public VerificationFactory( DeterminismAnalyzerConfig determinismAnalyzerConfig) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); - this.prestoActionFactory = requireNonNull(prestoActionFactory, "prestoActionFactory is null"); + this.queryActionsFactory = requireNonNull(queryActionsFactory, "queryActionsFactory is null"); this.queryRewriterFactory = requireNonNull(queryRewriterFactory, "queryRewriterFactory is null"); this.failureResolverManagerFactory = requireNonNull(failureResolverManagerFactory, "failureResolverManagerFactory is null"); - this.testResourceClient = requireNonNull(testResourceClient, "testResourceClient is null"); this.checksumValidator = requireNonNull(checksumValidator, "checksumValidator is null"); this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); this.verifierConfig = requireNonNull(verifierConfig, "config is null"); @@ -79,22 +74,18 @@ public Verification get(SourceQuery sourceQuery, Optional e switch (queryType.getCategory()) { case DATA_PRODUCING: VerificationContext verificationContext = existingContext.map(VerificationContext::createForResubmission).orElseGet(VerificationContext::create); - PrestoAction prestoAction = prestoActionFactory.create(sourceQuery, verificationContext); - QueryRewriter queryRewriter = queryRewriterFactory.create(prestoAction); + QueryActions queryActions = queryActionsFactory.create(sourceQuery, verificationContext); + QueryRewriter queryRewriter = queryRewriterFactory.create(queryActions.getHelperAction()); DeterminismAnalyzer determinismAnalyzer = new DeterminismAnalyzer( sourceQuery, - prestoAction, + queryActions.getHelperAction(), queryRewriter, checksumValidator, typeManager, - verificationContext, determinismAnalyzerConfig); - FailureResolverManager failureResolverManager = failureResolverManagerFactory.create(new FailureResolverFactoryContext( - sqlParser, - prestoAction, - testResourceClient)); + FailureResolverManager failureResolverManager = failureResolverManagerFactory.create(new FailureResolverFactoryContext(sqlParser, queryActions.getHelperAction())); return new DataVerification( - prestoAction, + queryActions, sourceQuery, queryRewriter, determinismAnalyzer, diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java index 4ec983773b6a8..1a11ea91608e6 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java @@ -47,6 +47,9 @@ public class VerifierConfig private boolean smartTeardown; private int verificationResubmissionLimit = 2; + private boolean setupOnMainClusters = true; + private boolean teardownOnMainClusters = true; + @NotNull public Optional> getWhitelist() { @@ -244,4 +247,30 @@ public VerifierConfig setVerificationResubmissionLimit(int verificationResubmiss this.verificationResubmissionLimit = verificationResubmissionLimit; return this; } + + public boolean isSetupOnMainClusters() + { + return setupOnMainClusters; + } + + @ConfigDescription("If true, run control/test setup queries on control/test clusters. Otherwise, run setup queries on the help cluster.") + @Config("setup-on-main-clusters") + public VerifierConfig setSetupOnMainClusters(boolean setupOnMainClusters) + { + this.setupOnMainClusters = setupOnMainClusters; + return this; + } + + public boolean isTeardownOnMainClusters() + { + return teardownOnMainClusters; + } + + @ConfigDescription("If true, run control/test teardown queries on control/test clusters. Otherwise, run teardown queries on the help cluster.") + @Config("teardown-on-main-clusters") + public VerifierConfig setTeardownOnMainClusters(boolean teardownOnMainClusters) + { + this.teardownOnMainClusters = teardownOnMainClusters; + return this; + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java index 01b63bc0ca153..db5f3dd644840 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java @@ -40,12 +40,7 @@ import com.facebook.presto.verifier.checksum.RowColumnValidator; import com.facebook.presto.verifier.checksum.SimpleColumnValidator; import com.facebook.presto.verifier.framework.Column.Category; -import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; -import com.facebook.presto.verifier.prestoaction.VerificationPrestoActionModule; import com.facebook.presto.verifier.resolver.FailureResolverModule; -import com.facebook.presto.verifier.retry.ForClusterConnection; -import com.facebook.presto.verifier.retry.ForPresto; -import com.facebook.presto.verifier.retry.RetryConfig; import com.facebook.presto.verifier.rewrite.VerificationQueryRewriterModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; @@ -83,16 +78,13 @@ public class VerifierModule { private final SqlParserOptions sqlParserOptions; private final List>> customQueryFilterClasses; - private final SqlExceptionClassifier exceptionClassifier; public VerifierModule( SqlParserOptions sqlParserOptions, - List>> customQueryFilterClasses, - SqlExceptionClassifier exceptionClassifier) + List>> customQueryFilterClasses) { this.sqlParserOptions = requireNonNull(sqlParserOptions, "sqlParserOptions is null"); this.customQueryFilterClasses = ImmutableList.copyOf(customQueryFilterClasses); - this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); } protected final void setup(Binder binder) @@ -104,9 +96,6 @@ protected final void setup(Binder binder) binder.bind(QueryConfigurationOverrides.class).annotatedWith(ForControl.class).to(Key.get(QueryConfigurationOverridesConfig.class, ForControl.class)).in(SINGLETON); binder.bind(QueryConfigurationOverrides.class).annotatedWith(ForTest.class).to(Key.get(QueryConfigurationOverridesConfig.class, ForTest.class)).in(SINGLETON); - configBinder(binder).bindConfig(RetryConfig.class, ForClusterConnection.class, "cluster-connection"); - configBinder(binder).bindConfig(RetryConfig.class, ForPresto.class, "presto"); - for (Class> customQueryFilterClass : customQueryFilterClasses) { binder.bind(customQueryFilterClass).in(SINGLETON); } @@ -137,7 +126,6 @@ protected final void setup(Binder binder) newSetBinder(binder, Type.class); // verifier - install(new VerificationPrestoActionModule(exceptionClassifier)); install(new VerificationQueryRewriterModule()); install(FailureResolverModule.BUILT_IN); binder.bind(VerificationManager.class).in(SINGLETON); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java index ae4422bf7bead..88c6aea2cc729 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java @@ -16,9 +16,9 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.common.type.TypeSignature; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.tree.Identifier; +import com.facebook.presto.verifier.event.QueryStatsEvent; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -44,25 +44,25 @@ public static Identifier delimitedIdentifier(String name) return new Identifier(name, true); } - public static void runAndConsume(Callable callable, Consumer queryStatsConsumer) + public static void runAndConsume(Callable callable, Consumer queryStatsConsumer) { runAndConsume(callable, queryStatsConsumer, e -> {}); } - public static void runAndConsume(Callable callable, Consumer queryStatsConsumer, Consumer queryExceptionConsumer) + public static void runAndConsume(Callable callable, Consumer queryStatsConsumer, Consumer queryExceptionConsumer) { callAndConsume(callable, identity(), queryStatsConsumer, queryExceptionConsumer); } - public static QueryResult callAndConsume(Callable> callable, Consumer queryStatsConsumer) + public static QueryResult callAndConsume(Callable> callable, Consumer queryStatsConsumer) { return callAndConsume(callable, QueryResult::getQueryStats, queryStatsConsumer, e -> {}); } private static V callAndConsume( Callable callable, - Function queryStatsTransformer, - Consumer queryStatsConsumer, + Function queryStatsTransformer, + Consumer queryStatsConsumer, Consumer queryExceptionConsumer) { try { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifyCommand.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifyCommand.java index 4fec728955cf0..bab18e13e959e 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifyCommand.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifyCommand.java @@ -32,6 +32,8 @@ public interface VerifyCommand Set getCustomEventClientTypes(); + Set getCustomQueryActionTypes(); + List>> getCustomQueryFilterClasses(); SqlExceptionClassifier getSqlExceptionClassifier(); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoAction.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoAction.java index ef96d6d227a26..c96897578e233 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoAction.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoAction.java @@ -17,6 +17,7 @@ import com.facebook.presto.jdbc.PrestoStatement; import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.ClusterConnectionException; import com.facebook.presto.verifier.framework.PrestoQueryException; import com.facebook.presto.verifier.framework.QueryConfiguration; @@ -35,22 +36,21 @@ import java.sql.ResultSet; import java.sql.SQLClientInfoException; import java.sql.SQLException; -import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.function.Consumer; -import static com.facebook.presto.SystemSessionProperties.QUERY_MAX_EXECUTION_TIME; -import static com.facebook.presto.SystemSessionProperties.QUERY_MAX_RUN_TIME; import static com.facebook.presto.sql.SqlFormatter.formatSql; -import static com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_MAIN; +import static com.facebook.presto.verifier.prestoaction.QueryActionUtil.mangleSessionProperties; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; public class JdbcPrestoAction implements PrestoAction { + public static final String QUERY_ACTION_TYPE = "presto-jdbc"; + private final SqlExceptionClassifier exceptionClassifier; private final QueryConfiguration queryConfiguration; @@ -66,17 +66,19 @@ public JdbcPrestoAction( SqlExceptionClassifier exceptionClassifier, QueryConfiguration queryConfiguration, VerificationContext verificationContext, - PrestoClusterConfig prestoClusterConfig, + PrestoActionConfig prestoActionConfig, + Duration metadataTimeout, + Duration checksumTimeout, @ForClusterConnection RetryConfig networkRetryConfig, @ForPresto RetryConfig prestoRetryConfig) { this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); this.queryConfiguration = requireNonNull(queryConfiguration, "queryConfiguration is null"); - this.jdbcUrl = requireNonNull(prestoClusterConfig.getJdbcUrl(), "jdbcUrl is null"); - this.queryTimeout = requireNonNull(prestoClusterConfig.getQueryTimeout(), "queryTimeout is null"); - this.metadataTimeout = requireNonNull(prestoClusterConfig.getMetadataTimeout(), "metadataTimeout is null"); - this.checksumTimeout = requireNonNull(prestoClusterConfig.getChecksumTimeout(), "checksumTimeout is null"); + this.jdbcUrl = requireNonNull(prestoActionConfig.getJdbcUrl(), "jdbcUrl is null"); + this.queryTimeout = requireNonNull(prestoActionConfig.getQueryTimeout(), "queryTimeout is null"); + this.metadataTimeout = requireNonNull(metadataTimeout, "metadataTimeout is null"); + this.checksumTimeout = requireNonNull(checksumTimeout, "checksumTimeout is null"); this.networkRetry = new RetryDriver<>( networkRetryConfig, @@ -91,7 +93,7 @@ public JdbcPrestoAction( } @Override - public QueryStats execute(Statement statement, QueryStage queryStage) + public QueryStatsEvent execute(Statement statement, QueryStage queryStage) { return execute(statement, queryStage, new NoResultStatementExecutor<>()); } @@ -145,17 +147,7 @@ private PrestoConnection getConnection(QueryStage queryStage) // Do nothing } - // configure session properties - Map sessionProperties = queryStage.isMain() || queryStage == DETERMINISM_ANALYSIS_MAIN - ? new HashMap<>(queryConfiguration.getSessionProperties()) - : new HashMap<>(); - - // Add or override query max execution time to enforce the timeout. - sessionProperties.put(QUERY_MAX_EXECUTION_TIME, getTimeout(queryStage).toString()); - - // Remove query max run time to respect execution time limit. - sessionProperties.remove(QUERY_MAX_RUN_TIME); - + Map sessionProperties = mangleSessionProperties(queryConfiguration.getSessionProperties(), queryStage, getTimeout(queryStage)); for (Entry entry : sessionProperties.entrySet()) { connection.setSessionProperty(entry.getKey(), entry.getValue()); } @@ -193,9 +185,9 @@ public synchronized void accept(QueryStats queryStats) this.queryStats = Optional.of(requireNonNull(queryStats, "queryStats is null")); } - public synchronized Optional getLastQueryStats() + public synchronized Optional getLastQueryStats() { - return queryStats; + return queryStats.map(stats -> new QueryStatsEvent(stats, Optional.empty())); } } @@ -240,12 +232,12 @@ public ProgressMonitor getProgressMonitor() } private static class NoResultStatementExecutor - implements StatementExecutor + implements StatementExecutor { private final ProgressMonitor progressMonitor = new ProgressMonitor(); @Override - public QueryStats execute(PrestoStatement statement, String query) + public QueryStatsEvent execute(PrestoStatement statement, String query) throws SQLException { boolean moreResults = statement.execute(query); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoActionFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoActionFactory.java new file mode 100644 index 0000000000000..afab7591ad141 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoActionFactory.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.verifier.framework.QueryConfiguration; +import com.facebook.presto.verifier.framework.VerificationContext; +import com.facebook.presto.verifier.retry.RetryConfig; +import io.airlift.units.Duration; + +import static java.util.Objects.requireNonNull; + +public class JdbcPrestoActionFactory + implements PrestoActionFactory +{ + private final SqlExceptionClassifier exceptionClassifier; + private final PrestoActionConfig prestoActionConfig; + private final RetryConfig networkRetryConfig; + private final RetryConfig prestoRetryConfig; + private final Duration metadataTimeout; + private final Duration checksumTimeout; + + public JdbcPrestoActionFactory( + SqlExceptionClassifier exceptionClassifier, + PrestoActionConfig prestoActionConfig, + RetryConfig networkRetryConfig, + RetryConfig prestoRetryConfig, + Duration metadataTimeout, + Duration checksumTimeout) + { + this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); + this.prestoActionConfig = requireNonNull(prestoActionConfig, "prestoClusterConfig is null"); + this.networkRetryConfig = requireNonNull(networkRetryConfig, "networkRetryConfig is null"); + this.prestoRetryConfig = requireNonNull(prestoRetryConfig, "prestoRetryConfig is null"); + this.metadataTimeout = requireNonNull(metadataTimeout, "metadataTimeout is null"); + this.checksumTimeout = requireNonNull(checksumTimeout, "checksumTimeout is null"); + } + + @Override + public JdbcPrestoAction create(QueryConfiguration queryConfiguration, VerificationContext verificationContext) + { + return new JdbcPrestoAction( + exceptionClassifier, + queryConfiguration, + verificationContext, + prestoActionConfig, + metadataTimeout, + checksumTimeout, + networkRetryConfig, + prestoRetryConfig); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAction.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAction.java index 622846f120dd6..4564090dd7a06 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAction.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAction.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.verifier.prestoaction; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.QueryResult; import com.facebook.presto.verifier.framework.QueryStage; @@ -27,6 +27,7 @@ import static java.util.Collections.unmodifiableList; public interface PrestoAction + extends QueryAction { @FunctionalInterface interface ResultSetConverter @@ -43,7 +44,7 @@ Optional apply(ResultSet resultSet) }; } - QueryStats execute(Statement statement, QueryStage queryStage); + QueryStatsEvent execute(Statement statement, QueryStage queryStage); QueryResult execute(Statement statement, QueryStage queryStage, ResultSetConverter converter); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoClusterConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionConfig.java similarity index 68% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoClusterConfig.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionConfig.java index 01f061ca54e32..deef2f7b5a466 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoClusterConfig.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionConfig.java @@ -26,20 +26,18 @@ import java.io.IOException; import java.util.Map; +import java.util.Optional; import static java.util.concurrent.TimeUnit.MINUTES; -public class PrestoClusterConfig +public class PrestoActionConfig implements PrestoAddress { private String host; private int jdbcPort; - private int httpPort; + private Optional httpPort = Optional.empty(); private Map jdbcUrlParameters = ImmutableMap.of(); - private Duration queryTimeout = new Duration(60, MINUTES); - private Duration metadataTimeout = new Duration(3, MINUTES); - private Duration checksumTimeout = new Duration(30, MINUTES); @Override @NotNull @@ -49,7 +47,7 @@ public String getHost() } @Config("host") - public PrestoClusterConfig setHost(String host) + public PrestoActionConfig setHost(String host) { this.host = host; return this; @@ -64,24 +62,23 @@ public int getJdbcPort() } @Config("jdbc-port") - public PrestoClusterConfig setJdbcPort(int jdbcPort) + public PrestoActionConfig setJdbcPort(int jdbcPort) { this.jdbcPort = jdbcPort; return this; } @Override - @Min(0) - @Max(65535) - public int getHttpPort() + @NotNull + public Optional getHttpPort() { return httpPort; } @Config("http-port") - public PrestoClusterConfig setHttpPort(int httpPort) + public PrestoActionConfig setHttpPort(Integer httpPort) { - this.httpPort = httpPort; + this.httpPort = Optional.ofNullable(httpPort); return this; } @@ -93,7 +90,7 @@ public Map getJdbcUrlParameters() } @Config("jdbc-url-parameters") - public PrestoClusterConfig setJdbcUrlParameters(String jdbcUrlParameters) + public PrestoActionConfig setJdbcUrlParameters(String jdbcUrlParameters) { if (jdbcUrlParameters == null) { return this; @@ -115,35 +112,9 @@ public Duration getQueryTimeout() } @Config("query-timeout") - public PrestoClusterConfig setQueryTimeout(Duration queryTimeout) + public PrestoActionConfig setQueryTimeout(Duration queryTimeout) { this.queryTimeout = queryTimeout; return this; } - - @MinDuration("1s") - public Duration getMetadataTimeout() - { - return metadataTimeout; - } - - @Config("metadata-timeout") - public PrestoClusterConfig setMetadataTimeout(Duration metadataTimeout) - { - this.metadataTimeout = metadataTimeout; - return this; - } - - @MinDuration("1s") - public Duration getChecksumTimeout() - { - return checksumTimeout; - } - - @Config("checksum-timeout") - public PrestoClusterConfig setChecksumTimeout(Duration checksumTimeout) - { - this.checksumTimeout = checksumTimeout; - return this; - } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionFactory.java index 1f29a4941ca4d..ecc366d571394 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionFactory.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionFactory.java @@ -13,10 +13,12 @@ */ package com.facebook.presto.verifier.prestoaction; -import com.facebook.presto.verifier.framework.SourceQuery; +import com.facebook.presto.verifier.framework.QueryConfiguration; import com.facebook.presto.verifier.framework.VerificationContext; public interface PrestoActionFactory + extends QueryActionFactory { - PrestoAction create(SourceQuery sourceQuery, VerificationContext verificationContext); + @Override + PrestoAction create(QueryConfiguration queryConfiguration, VerificationContext verificationContext); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAddress.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAddress.java index d3b93c7184cc5..3900cd79cf026 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAddress.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAddress.java @@ -15,7 +15,9 @@ import java.net.URI; import java.util.Map; +import java.util.Optional; +import static com.google.common.base.Preconditions.checkState; import static java.lang.String.format; import static java.util.stream.Collectors.joining; @@ -25,7 +27,7 @@ public interface PrestoAddress int getJdbcPort(); - int getHttpPort(); + Optional getHttpPort(); Map getJdbcUrlParameters(); @@ -39,6 +41,7 @@ default String getJdbcUrl() default URI getHttpUri(String path) { - return URI.create(format("http://%s:%s", getHost(), getHttpPort())).resolve(path); + checkState(getHttpPort().isPresent(), "httpPort is not present"); + return URI.create(format("http://%s:%s", getHost(), getHttpPort().get())).resolve(path); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java index e7bf34eedc3a9..68da4083279d8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java @@ -15,10 +15,10 @@ import com.facebook.presto.connector.thrift.ThriftErrorCode; import com.facebook.presto.hive.HiveErrorCode; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.plugin.jdbc.JdbcErrorCode; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.StandardErrorCode; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.ClusterConnectionException; import com.facebook.presto.verifier.framework.PrestoQueryException; import com.facebook.presto.verifier.framework.QueryException; @@ -143,20 +143,31 @@ public static Builder defaultBuilder() .addResubmittedError(SYNTAX_ERROR, Optional.of(TEST_SETUP), Optional.of(TABLE_ALREADY_EXISTS_PATTERN)); } - public QueryException createException(QueryStage queryStage, Optional queryStats, SQLException cause) + public QueryException createException(QueryStage queryStage, Optional queryStats, SQLException cause) { Optional clusterConnectionExceptionCause = getClusterConnectionExceptionCause(cause); if (clusterConnectionExceptionCause.isPresent()) { return new ClusterConnectionException(clusterConnectionExceptionCause.get(), queryStage); } - Optional errorCode = Optional.ofNullable(errorByCode.get(cause.getErrorCode())); - boolean retryable = errorCode.isPresent() - && (retryableErrors.contains(errorCode.get()) - || conditionalRetryableErrors.stream().anyMatch(matcher -> matcher.matches(errorCode.get(), queryStage, cause.getMessage()))); + Optional errorCode = getErrorCode(cause.getErrorCode()); + boolean retryable = errorCode.isPresent() && isRetryable(errorCode.get(), queryStage, cause.getMessage()); return new PrestoQueryException(cause, retryable, queryStage, errorCode, queryStats); } + @Override + public Optional getErrorCode(int code) + { + return Optional.ofNullable(errorByCode.get(code)); + } + + @Override + public boolean isRetryable(ErrorCodeSupplier errorCode, QueryStage queryStage, String message) + { + return retryableErrors.contains(errorCode) + || conditionalRetryableErrors.stream().anyMatch(matcher -> matcher.matches(errorCode, queryStage, message)); + } + public boolean shouldResubmit(Throwable throwable) { if (!(throwable instanceof PrestoQueryException)) { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryAction.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryAction.java new file mode 100644 index 0000000000000..8241675a522a6 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryAction.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.verifier.event.QueryStatsEvent; +import com.facebook.presto.verifier.framework.QueryStage; + +public interface QueryAction +{ + QueryStatsEvent execute(Statement statement, QueryStage queryStage); +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionFactory.java new file mode 100644 index 0000000000000..25f4ca6f4d929 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionFactory.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.verifier.framework.QueryConfiguration; +import com.facebook.presto.verifier.framework.VerificationContext; + +public interface QueryActionFactory +{ + QueryAction create(QueryConfiguration queryConfiguration, VerificationContext verificationContext); +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionUtil.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionUtil.java new file mode 100644 index 0000000000000..20eb458586337 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionUtil.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.verifier.framework.QueryStage; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; + +import java.util.HashMap; +import java.util.Map; + +import static com.facebook.presto.SystemSessionProperties.QUERY_MAX_EXECUTION_TIME; +import static com.facebook.presto.SystemSessionProperties.QUERY_MAX_RUN_TIME; +import static com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_MAIN; + +public class QueryActionUtil +{ + private QueryActionUtil() + { + } + + public static Map mangleSessionProperties( + Map mainQuerySessionProperty, + QueryStage queryStage, + Duration queryTimeout) + { + // configure session properties + Map sessionProperties = queryStage.isMain() || queryStage == DETERMINISM_ANALYSIS_MAIN + ? new HashMap<>(mainQuerySessionProperty) + : new HashMap<>(); + + // Add or override query max execution time to enforce the timeout. + sessionProperties.put(QUERY_MAX_EXECUTION_TIME, queryTimeout.toString()); + + // Remove query max run time to respect execution time limit. + sessionProperties.remove(QUERY_MAX_RUN_TIME); + + return ImmutableMap.copyOf(sessionProperties); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActions.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActions.java new file mode 100644 index 0000000000000..9a7ad5df866aa --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActions.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import static java.util.Objects.requireNonNull; + +public class QueryActions +{ + private final PrestoAction helperAction; + private final QueryAction controlAction; + private final QueryAction testAction; + + public QueryActions(PrestoAction helperAction, QueryAction controlAction, QueryAction testAction) + { + this.helperAction = requireNonNull(helperAction, "helperAction is null"); + this.controlAction = requireNonNull(controlAction, "controlAction is null"); + this.testAction = requireNonNull(testAction, "testAction is null"); + } + + public PrestoAction getHelperAction() + { + return helperAction; + } + + public QueryAction getControlAction() + { + return controlAction; + } + + public QueryAction getTestAction() + { + return testAction; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsConfig.java new file mode 100644 index 0000000000000..6691c843da4a9 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsConfig.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.airlift.configuration.Config; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.NotNull; + +import static java.util.concurrent.TimeUnit.MINUTES; + +public class QueryActionsConfig +{ + private String controlQueryActionType = JdbcPrestoAction.QUERY_ACTION_TYPE; + private String testQueryActionType = JdbcPrestoAction.QUERY_ACTION_TYPE; + private boolean runHelperQueriesOnControl = true; + + private Duration metadataTimeout = new Duration(3, MINUTES); + private Duration checksumTimeout = new Duration(30, MINUTES); + + @NotNull + public String getControlQueryActionType() + { + return controlQueryActionType; + } + + @Config("control.query-action-type") + public QueryActionsConfig setControlQueryActionType(String controlQueryActionType) + { + this.controlQueryActionType = controlQueryActionType; + return this; + } + + @NotNull + public String getTestQueryActionType() + { + return testQueryActionType; + } + + @Config("test.query-action-type") + public QueryActionsConfig setTestQueryActionType(String testQueryActionType) + { + this.testQueryActionType = testQueryActionType; + return this; + } + + public boolean isRunHelperQueriesOnControl() + { + return runHelperQueriesOnControl; + } + + @Config("run-helper-queries-on-control") + public QueryActionsConfig setRunHelperQueriesOnControl(boolean runHelperQueriesOnControl) + { + this.runHelperQueriesOnControl = runHelperQueriesOnControl; + return this; + } + + @MinDuration("1s") + public Duration getMetadataTimeout() + { + return metadataTimeout; + } + + @Config("metadata-timeout") + public QueryActionsConfig setMetadataTimeout(Duration metadataTimeout) + { + this.metadataTimeout = metadataTimeout; + return this; + } + + @MinDuration("1s") + public Duration getChecksumTimeout() + { + return checksumTimeout; + } + + @Config("checksum-timeout") + public QueryActionsConfig setChecksumTimeout(Duration checksumTimeout) + { + this.checksumTimeout = checksumTimeout; + return this; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsFactory.java new file mode 100644 index 0000000000000..342556f51e826 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsFactory.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.verifier.framework.SourceQuery; +import com.facebook.presto.verifier.framework.VerificationContext; + +public interface QueryActionsFactory +{ + QueryActions create(SourceQuery sourceQuery, VerificationContext verificationContext); +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsModule.java new file mode 100644 index 0000000000000..5fbe28ab302d3 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsModule.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.presto.verifier.annotation.ForControl; +import com.facebook.presto.verifier.annotation.ForHelper; +import com.facebook.presto.verifier.annotation.ForTest; +import com.facebook.presto.verifier.retry.ForClusterConnection; +import com.facebook.presto.verifier.retry.ForPresto; +import com.facebook.presto.verifier.retry.RetryConfig; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Binder; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; + +import javax.inject.Provider; + +import java.lang.annotation.Annotation; +import java.util.Set; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.inject.Scopes.SINGLETON; +import static java.util.Objects.requireNonNull; + +public class QueryActionsModule + extends AbstractConfigurationAwareModule +{ + private final SqlExceptionClassifier exceptionClassifier; + private final Set supportedQueryActionTypes; + + public QueryActionsModule( + SqlExceptionClassifier exceptionClassifier, + Set customQueryActionTypes) + { + this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); + this.supportedQueryActionTypes = ImmutableSet.builder() + .add(JdbcPrestoAction.QUERY_ACTION_TYPE) + .addAll(customQueryActionTypes) + .build(); + } + + @Override + protected void setup(Binder binder) + { + binder.bind(SqlExceptionClassifier.class).toInstance(exceptionClassifier); + binder.bind(QueryActionsFactory.class).to(QueryActionsProvider.class).in(SINGLETON); + + configBinder(binder).bindConfig(RetryConfig.class, ForClusterConnection.class, "cluster-connection"); + configBinder(binder).bindConfig(RetryConfig.class, ForPresto.class, "presto"); + + QueryActionsConfig config = buildConfigObject(QueryActionsConfig.class); + String controlQueryActionType = config.getControlQueryActionType(); + String testQueryActionType = config.getTestQueryActionType(); + checkArgument(supportedQueryActionTypes.contains(controlQueryActionType), "Unsupported QueryAction type: %s", controlQueryActionType); + checkArgument(supportedQueryActionTypes.contains(testQueryActionType), "Unsupported QueryAction type: %s", testQueryActionType); + + if (controlQueryActionType.equals(JdbcPrestoAction.QUERY_ACTION_TYPE)) { + configBinder(binder).bindConfig(PrestoActionConfig.class, ForControl.class, "control"); + binder.bind(QueryActionFactory.class).annotatedWith(ForControl.class).toProvider(new JdbcPrestoActionFactoryProvider(ForControl.class)).in(SINGLETON); + } + if (testQueryActionType.equals(JdbcPrestoAction.QUERY_ACTION_TYPE)) { + configBinder(binder).bindConfig(PrestoActionConfig.class, ForTest.class, "test"); + binder.bind(QueryActionFactory.class).annotatedWith(ForTest.class).toProvider(new JdbcPrestoActionFactoryProvider(ForTest.class)).in(SINGLETON); + } + if (config.isRunHelperQueriesOnControl()) { + checkArgument(controlQueryActionType.equals(JdbcPrestoAction.QUERY_ACTION_TYPE), "Cannot run helper queries on control cluster because it is not a presto-jdbc action"); + binder.bind(PrestoActionFactory.class).annotatedWith(ForHelper.class).toProvider(new JdbcPrestoActionFactoryProvider(ForControl.class)).in(SINGLETON); + } + else { + configBinder(binder).bindConfig(PrestoActionConfig.class, ForHelper.class, "helper"); + binder.bind(PrestoActionFactory.class).annotatedWith(ForHelper.class).toProvider(new JdbcPrestoActionFactoryProvider(ForHelper.class)).in(SINGLETON); + } + } + + public static class JdbcPrestoActionFactoryProvider + implements Provider + { + private final Class annotationClass; + private Injector injector; + + public JdbcPrestoActionFactoryProvider(Class annotationClass) + { + this.annotationClass = requireNonNull(annotationClass, "annotationClass is null"); + } + + @Inject + public void setInjector(Injector injector) + { + this.injector = injector; + } + + @Override + public PrestoActionFactory get() + { + QueryActionsConfig queryActionsConfig = injector.getInstance(QueryActionsConfig.class); + return new JdbcPrestoActionFactory( + injector.getInstance(SqlExceptionClassifier.class), + injector.getInstance(Key.get(PrestoActionConfig.class, annotationClass)), + injector.getInstance(Key.get(RetryConfig.class, ForClusterConnection.class)), + injector.getInstance(Key.get(RetryConfig.class, ForPresto.class)), + queryActionsConfig.getMetadataTimeout(), + queryActionsConfig.getChecksumTimeout()); + } + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsProvider.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsProvider.java new file mode 100644 index 0000000000000..fc47a0c01ddfb --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.verifier.annotation.ForControl; +import com.facebook.presto.verifier.annotation.ForHelper; +import com.facebook.presto.verifier.annotation.ForTest; +import com.facebook.presto.verifier.framework.SourceQuery; +import com.facebook.presto.verifier.framework.VerificationContext; +import com.google.inject.Inject; + +import static java.util.Objects.requireNonNull; + +public class QueryActionsProvider + implements QueryActionsFactory +{ + private final PrestoActionFactory helpActionFactory; + private final QueryActionFactory controlActionFactory; + private final QueryActionFactory testActionFactory; + + @Inject + public QueryActionsProvider( + @ForHelper PrestoActionFactory helpActionFactory, + @ForControl QueryActionFactory controlActionFactory, + @ForTest QueryActionFactory testActionFactory) + { + this.helpActionFactory = requireNonNull(helpActionFactory, "helpActionFactory is null"); + this.controlActionFactory = requireNonNull(controlActionFactory, "controlActionFactory is null"); + this.testActionFactory = requireNonNull(testActionFactory, "testActionFactory is null"); + } + + public QueryActions create(SourceQuery sourceQuery, VerificationContext verificationContext) + { + return new QueryActions( + helpActionFactory.create(sourceQuery.getControlConfiguration(), verificationContext), + controlActionFactory.create(sourceQuery.getControlConfiguration(), verificationContext), + testActionFactory.create(sourceQuery.getTestConfiguration(), verificationContext)); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoAction.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoAction.java deleted file mode 100644 index 6dd68e28ff9b8..0000000000000 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoAction.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.verifier.prestoaction; - -import com.facebook.presto.jdbc.QueryStats; -import com.facebook.presto.sql.tree.Statement; -import com.facebook.presto.verifier.framework.ClusterType; -import com.facebook.presto.verifier.framework.QueryResult; -import com.facebook.presto.verifier.framework.QueryStage; -import com.google.common.collect.ImmutableMap; - -import java.util.Map; - -import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; -import static com.facebook.presto.verifier.framework.ClusterType.TEST; - -public class RoutingPrestoAction - implements PrestoAction -{ - private final Map prestoActions; - - public RoutingPrestoAction(PrestoAction controlAction, PrestoAction testAction) - { - this.prestoActions = ImmutableMap.of( - CONTROL, controlAction, - TEST, testAction); - } - - @Override - public QueryStats execute(Statement statement, QueryStage queryStage) - { - return prestoActions.get(queryStage.getTargetCluster()).execute(statement, queryStage); - } - - @Override - public QueryResult execute(Statement statement, QueryStage queryStage, ResultSetConverter converter) - { - return prestoActions.get(queryStage.getTargetCluster()).execute(statement, queryStage, converter); - } -} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoActionFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoActionFactory.java deleted file mode 100644 index 59b678b2d582e..0000000000000 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoActionFactory.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.verifier.prestoaction; - -import com.facebook.presto.verifier.annotation.ForControl; -import com.facebook.presto.verifier.annotation.ForTest; -import com.facebook.presto.verifier.framework.SourceQuery; -import com.facebook.presto.verifier.framework.VerificationContext; -import com.facebook.presto.verifier.retry.ForClusterConnection; -import com.facebook.presto.verifier.retry.ForPresto; -import com.facebook.presto.verifier.retry.RetryConfig; - -import javax.inject.Inject; - -import static java.util.Objects.requireNonNull; - -public class RoutingPrestoActionFactory - implements PrestoActionFactory -{ - private final SqlExceptionClassifier exceptionClassifier; - private final PrestoClusterConfig controlClusterConfig; - private final PrestoClusterConfig testClusterConfig; - private final RetryConfig networkRetryConfig; - private final RetryConfig prestoRetryConfig; - - @Inject - public RoutingPrestoActionFactory( - SqlExceptionClassifier exceptionClassifier, - @ForControl PrestoClusterConfig controlClusterConfig, - @ForTest PrestoClusterConfig testClusterConfig, - @ForClusterConnection RetryConfig networkRetryConfig, - @ForPresto RetryConfig prestoRetryConfig) - { - this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); - this.controlClusterConfig = requireNonNull(controlClusterConfig, "controlClusterConfig is null"); - this.testClusterConfig = requireNonNull(testClusterConfig, "testClusterConfig is null"); - this.networkRetryConfig = requireNonNull(networkRetryConfig, "networkRetryConfig is null"); - this.prestoRetryConfig = requireNonNull(prestoRetryConfig, "verifierConfig is null"); - } - - @Override - public RoutingPrestoAction create(SourceQuery sourceQuery, VerificationContext verificationContext) - { - return new RoutingPrestoAction( - new JdbcPrestoAction( - exceptionClassifier, - sourceQuery.getControlConfiguration(), - verificationContext, - controlClusterConfig, - networkRetryConfig, - prestoRetryConfig), - new JdbcPrestoAction( - exceptionClassifier, - sourceQuery.getTestConfiguration(), - verificationContext, - testClusterConfig, - networkRetryConfig, - prestoRetryConfig)); - } -} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/SqlExceptionClassifier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/SqlExceptionClassifier.java index dbbef8c078647..b21b8799e91c8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/SqlExceptionClassifier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/SqlExceptionClassifier.java @@ -13,7 +13,8 @@ */ package com.facebook.presto.verifier.prestoaction; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.QueryException; import com.facebook.presto.verifier.framework.QueryStage; @@ -22,7 +23,11 @@ public interface SqlExceptionClassifier { - QueryException createException(QueryStage queryStage, Optional queryStats, SQLException cause); + QueryException createException(QueryStage queryStage, Optional queryStats, SQLException cause); + + Optional getErrorCode(int code); + + boolean isRetryable(ErrorCodeSupplier errorCode, QueryStage queryStage, String message); boolean shouldResubmit(Throwable throwable); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ChecksumExceededTimeLimitFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ChecksumExceededTimeLimitFailureResolver.java index 1b1d50bc63436..0e6f54605cc3e 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ChecksumExceededTimeLimitFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ChecksumExceededTimeLimitFailureResolver.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.resolver; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; import com.google.common.collect.ImmutableSet; @@ -30,7 +30,7 @@ public class ChecksumExceededTimeLimitFailureResolver public static final String NAME = "checksum-exceeded-time-limit"; @Override - public Optional resolveQueryFailure(QueryStats controlQueryStats, QueryException queryException, Optional test) + public Optional resolveQueryFailure(QueryStatsEvent controlQueryStats, QueryException queryException, Optional test) { return mapMatchingPrestoException(queryException, CONTROL_CHECKSUM, ImmutableSet.of(EXCEEDED_TIME_LIMIT), e -> Optional.of("Time limit exceeded when running control checksum query")); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/HttpNodeResourceClient.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeFetcher.java similarity index 78% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/HttpNodeResourceClient.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeFetcher.java index 5fe405209bf88..c65137e7808de 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/HttpNodeResourceClient.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeFetcher.java @@ -11,18 +11,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.prestoaction; +package com.facebook.presto.verifier.resolver; import com.facebook.airlift.http.client.HttpClient; import com.facebook.airlift.http.client.Request; import com.facebook.airlift.json.ObjectMapperProvider; +import com.facebook.presto.verifier.prestoaction.PrestoActionConfig; +import com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier; import com.facebook.presto.verifier.retry.RetryConfig; import com.facebook.presto.verifier.retry.RetryDriver; import com.fasterxml.jackson.core.type.TypeReference; -import com.google.inject.Inject; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.URI; import java.util.List; import java.util.Map; @@ -30,40 +32,40 @@ import static com.facebook.airlift.http.client.StringResponseHandler.StringResponse; import static com.facebook.airlift.http.client.StringResponseHandler.createStringResponseHandler; import static com.google.common.base.Preconditions.checkState; -import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE; import static javax.ws.rs.core.MediaType.APPLICATION_JSON; import static javax.ws.rs.core.Response.Status.OK; -public class HttpNodeResourceClient - implements NodeResourceClient +public class ClusterSizeFetcher + implements ClusterSizeSupplier { + private static final String PATH = "/v1/node"; + private final HttpClient httpClient; - private final PrestoAddress prestoAddress; + private final URI nodeResourceUri; private final RetryDriver networkRetry; - @Inject - public HttpNodeResourceClient( + public ClusterSizeFetcher( HttpClient httpClient, - PrestoClusterConfig prestoAddress, + PrestoActionConfig prestoAddress, RetryConfig networkRetryConfig) { this.httpClient = requireNonNull(httpClient, "httpClient is null"); - this.prestoAddress = requireNonNull(prestoAddress, "prestoAddress is null"); + this.nodeResourceUri = requireNonNull(prestoAddress.getHttpUri(PATH), "nodeResourceUri is null"); this.networkRetry = new RetryDriver<>(networkRetryConfig, PrestoExceptionClassifier::isClusterConnectionException, RuntimeException.class, e -> {}); } @Override - public int getClusterSize(String path) + public int getClusterSize() { - return networkRetry.run(format("getJsonResponse()", path), () -> getClusterSizeOnce(path)); + return networkRetry.run("fetchClusterSize", this::fetchClusterSize); } - private int getClusterSizeOnce(String path) + private int fetchClusterSize() { Request request = prepareGet() - .setUri(prestoAddress.getHttpUri(path)) + .setUri(nodeResourceUri) .setHeader(CONTENT_TYPE, APPLICATION_JSON) .build(); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/VerificationPrestoActionModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeFetcherModule.java similarity index 53% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/VerificationPrestoActionModule.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeFetcherModule.java index 41e7f67cc6350..bfe11158af725 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/VerificationPrestoActionModule.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeFetcherModule.java @@ -11,11 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.prestoaction; +package com.facebook.presto.verifier.resolver; import com.facebook.airlift.http.client.HttpClient; -import com.facebook.presto.verifier.annotation.ForControl; import com.facebook.presto.verifier.annotation.ForTest; +import com.facebook.presto.verifier.prestoaction.PrestoActionConfig; import com.facebook.presto.verifier.retry.ForClusterConnection; import com.facebook.presto.verifier.retry.RetryConfig; import com.google.inject.Binder; @@ -23,40 +23,25 @@ import com.google.inject.Provides; import com.google.inject.Singleton; -import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; -import static com.google.inject.Scopes.SINGLETON; -import static java.util.Objects.requireNonNull; -public class VerificationPrestoActionModule +public class ClusterSizeFetcherModule implements Module { - private final SqlExceptionClassifier exceptionClassifier; - - public VerificationPrestoActionModule(SqlExceptionClassifier exceptionClassifier) - { - this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); - } - @Override public void configure(Binder binder) { - configBinder(binder).bindConfig(PrestoClusterConfig.class, ForControl.class, "control"); - configBinder(binder).bindConfig(PrestoClusterConfig.class, ForTest.class, "test"); - httpClientBinder(binder).bindHttpClient("test", ForTest.class); - binder.bind(PrestoActionFactory.class).to(RoutingPrestoActionFactory.class).in(SINGLETON); - binder.bind(SqlExceptionClassifier.class).toInstance(exceptionClassifier); } @Provides @ForTest @Singleton - public NodeResourceClient providesTestHttpPrestoResourceClient( + public ClusterSizeSupplier providesTestClusterSizeSupplier( @ForTest HttpClient httpClient, - @ForTest PrestoClusterConfig prestoClusterConfig, + @ForTest PrestoActionConfig prestoActionConfig, @ForClusterConnection RetryConfig networkRetryConfig) { - return new HttpNodeResourceClient(httpClient, prestoClusterConfig, networkRetryConfig); + return new ClusterSizeFetcher(httpClient, prestoActionConfig, networkRetryConfig); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/NodeResourceClient.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeSupplier.java similarity index 82% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/NodeResourceClient.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeSupplier.java index 599a7f6c81279..44d9c379d535b 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/NodeResourceClient.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ClusterSizeSupplier.java @@ -11,9 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.prestoaction; +package com.facebook.presto.verifier.resolver; -public interface NodeResourceClient +public interface ClusterSizeSupplier { - int getClusterSize(String path); + int getClusterSize(); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededGlobalMemoryLimitFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededGlobalMemoryLimitFailureResolver.java index b4811d311930e..48a6e6b80ff36 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededGlobalMemoryLimitFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededGlobalMemoryLimitFailureResolver.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.resolver; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; import com.google.common.collect.ImmutableSet; @@ -30,7 +30,7 @@ public class ExceededGlobalMemoryLimitFailureResolver public static final String NAME = "exceeded-global-memory-limit"; @Override - public Optional resolveQueryFailure(QueryStats controlQueryStats, QueryException queryException, Optional test) + public Optional resolveQueryFailure(QueryStatsEvent controlQueryStats, QueryException queryException, Optional test) { return mapMatchingPrestoException(queryException, TEST_MAIN, ImmutableSet.of(EXCEEDED_GLOBAL_MEMORY_LIMIT), e -> e.getQueryStats().isPresent() && controlQueryStats.getPeakTotalMemoryBytes() > e.getQueryStats().get().getPeakTotalMemoryBytes() diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java index 433f9ac998a72..604a324a34b5d 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.resolver; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; import com.google.common.collect.ImmutableSet; @@ -30,7 +30,7 @@ public class ExceededTimeLimitFailureResolver public static final String NAME = "exceeded-time-limit"; @Override - public Optional resolveQueryFailure(QueryStats controlQueryStats, QueryException queryException, Optional test) + public Optional resolveQueryFailure(QueryStatsEvent controlQueryStats, QueryException queryException, Optional test) { return mapMatchingPrestoException(queryException, TEST_MAIN, ImmutableSet.of(EXCEEDED_TIME_LIMIT), e -> Optional.of("Time limit exceeded on test cluster")); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolver.java index 0474fc0a1919e..b823c955fa916 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolver.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.resolver; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.MatchResult; import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; @@ -22,7 +22,7 @@ public interface FailureResolver { - default Optional resolveQueryFailure(QueryStats controlQueryStats, QueryException queryException, Optional test) + default Optional resolveQueryFailure(QueryStatsEvent controlQueryStats, QueryException queryException, Optional test) { return Optional.empty(); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactoryContext.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactoryContext.java index e45c2ccf8f6e3..8df8702b0b632 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactoryContext.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactoryContext.java @@ -14,7 +14,6 @@ package com.facebook.presto.verifier.resolver; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.verifier.prestoaction.NodeResourceClient; import com.facebook.presto.verifier.prestoaction.PrestoAction; import static java.util.Objects.requireNonNull; @@ -23,16 +22,13 @@ public class FailureResolverFactoryContext { private final SqlParser sqlParser; private final PrestoAction prestoAction; - private final NodeResourceClient testResourceClient; public FailureResolverFactoryContext( SqlParser sqlParser, - PrestoAction prestoAction, - NodeResourceClient testResourceClient) + PrestoAction prestoAction) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); - this.testResourceClient = requireNonNull(testResourceClient, "testResourceClient is null"); } public SqlParser getSqlParser() @@ -44,9 +40,4 @@ public PrestoAction getPrestoAction() { return prestoAction; } - - public NodeResourceClient getTestResourceClient() - { - return testResourceClient; - } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManager.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManager.java index b4e4d2d9220ba..f1b08c860b800 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManager.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManager.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.resolver; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.MatchResult; import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; @@ -32,7 +32,7 @@ public FailureResolverManager(Set failureResolvers) this.failureResolvers = requireNonNull(failureResolvers, "failureResolvers is null"); } - public Optional resolveException(QueryStats controlQueryStats, Throwable throwable, Optional test) + public Optional resolveException(QueryStatsEvent controlQueryStats, Throwable throwable, Optional test) { if (!(throwable instanceof QueryException)) { return Optional.of("Verifier Error"); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverModule.java index 841c97755d3ea..8f7f630c6940d 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverModule.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverModule.java @@ -16,6 +16,7 @@ import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; +import com.google.inject.Module; import java.util.ArrayList; import java.util.List; @@ -36,7 +37,11 @@ public class FailureResolverModule .bind(ExceededTimeLimitFailureResolver.NAME, ExceededTimeLimitFailureResolver.class, Optional.empty()) .bind(ChecksumExceededTimeLimitFailureResolver.NAME, ChecksumExceededTimeLimitFailureResolver.class, Optional.empty()) .bind(VerifierLimitationFailureResolver.NAME, VerifierLimitationFailureResolver.class, Optional.empty()) - .bindFactory(TooManyOpenPartitionsFailureResolver.NAME, TooManyOpenPartitionsFailureResolver.Factory.class, Optional.of(TooManyOpenPartitionsFailureResolverConfig.class)) + .bindFactory( + TooManyOpenPartitionsFailureResolver.NAME, + TooManyOpenPartitionsFailureResolver.Factory.class, + Optional.of(TooManyOpenPartitionsFailureResolverConfig.class), + ImmutableList.of(new ClusterSizeFetcherModule())) // Result Mismatch Resolvers .bind(StructuredColumnMismatchResolver.NAME, StructuredColumnMismatchResolver.class, Optional.empty()) .bind(IgnoredFunctionsMismatchResolver.NAME, IgnoredFunctionsMismatchResolver.class, Optional.of(IgnoredFunctionsMismatchResolverConfig.class)) @@ -59,14 +64,16 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(FailureResolverConfig.class); binder.bind(FailureResolverManagerFactory.class).in(SINGLETON); if (!buildConfigObject(FailureResolverConfig.class).isEnabled()) { + newSetBinder(binder, FailureResolver.class); + newSetBinder(binder, FailureResolverFactory.class); return; } for (FailureResolverBinding binding : resolvers) { - bind(binder, binding.getName(), Optional.of(binding.getResolverClass()), Optional.empty(), binding.getConfigClass()); + bind(binder, binding.getName(), Optional.of(binding.getResolverClass()), Optional.empty(), binding.getConfigClass(), binding.getAdditionalModules()); } for (FailureResolverFactoryBinding binding : factories) { - bind(binder, binding.getName(), Optional.empty(), Optional.of(binding.getFactoryClass()), binding.getConfigClass()); + bind(binder, binding.getName(), Optional.empty(), Optional.of(binding.getFactoryClass()), binding.getConfigClass(), binding.getAdditionalModules()); } } @@ -75,13 +82,15 @@ private void bind( String name, Optional> failureResolverClass, Optional> failureResolverFactoryClass, - Optional> failureResolverConfigClass) + Optional> failureResolverConfigClass, + List additionalModules) { configBinder(binder).bindConfig(FailureResolverConfig.class, named(name), name); if (buildConfigObject(FailureResolverConfig.class, name).isEnabled()) { failureResolverClass.ifPresent(clazz -> newSetBinder(binder, FailureResolver.class).addBinding().to(clazz).in(SINGLETON)); failureResolverFactoryClass.ifPresent(clazz -> newSetBinder(binder, FailureResolverFactory.class).addBinding().to(clazz).in(SINGLETON)); failureResolverConfigClass.ifPresent(clazz -> configBinder(binder).bindConfig(clazz, name)); + additionalModules.forEach(this::install); } } @@ -101,9 +110,13 @@ public Builder bind(String name, Class failureResolve return this; } - public Builder bindFactory(String name, Class failureResolverFactoryClass, Optional> configClass) + public Builder bindFactory( + String name, + Class failureResolverFactoryClass, + Optional> configClass, + List additionalModules) { - factories.add(new FailureResolverFactoryBinding(name, failureResolverFactoryClass, configClass)); + factories.add(new FailureResolverFactoryBinding(name, failureResolverFactoryClass, configClass, additionalModules)); return this; } @@ -120,7 +133,7 @@ private static class FailureResolverBinding public FailureResolverBinding(String name, Class resolverClass, Optional> configClass) { - super(name, configClass); + super(name, configClass, ImmutableList.of()); this.resolverClass = resolverClass; } @@ -135,9 +148,9 @@ private static class FailureResolverFactoryBinding { private final Class factoryClass; - public FailureResolverFactoryBinding(String name, Class factoryClass, Optional> configClass) + public FailureResolverFactoryBinding(String name, Class factoryClass, Optional> configClass, List additionalModules) { - super(name, configClass); + super(name, configClass, additionalModules); this.factoryClass = factoryClass; } @@ -151,11 +164,13 @@ private abstract static class AbstractFailureResolverBinding { private final String name; private final Optional> configClass; + private final List additionalModules; - public AbstractFailureResolverBinding(String name, Optional> configClass) + public AbstractFailureResolverBinding(String name, Optional> configClass, List additionalModules) { this.name = requireNonNull(name, "name is null"); this.configClass = requireNonNull(configClass, "configClass is null"); + this.additionalModules = ImmutableList.copyOf(additionalModules); } public String getName() @@ -167,5 +182,10 @@ public Optional> getConfigClass() { return configClass; } + + public List getAdditionalModules() + { + return additionalModules; + } } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/TooManyOpenPartitionsFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/TooManyOpenPartitionsFailureResolver.java index fa23f9d64b450..67f7e4932149c 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/TooManyOpenPartitionsFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/TooManyOpenPartitionsFailureResolver.java @@ -14,13 +14,14 @@ package com.facebook.presto.verifier.resolver; import com.facebook.airlift.log.Logger; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.LongLiteral; import com.facebook.presto.sql.tree.Property; import com.facebook.presto.sql.tree.ShowCreate; +import com.facebook.presto.verifier.annotation.ForTest; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; import com.facebook.presto.verifier.prestoaction.PrestoAction; @@ -52,13 +53,13 @@ public class TooManyOpenPartitionsFailureResolver public static final String NAME = "too-many-open-partitions"; private static final Logger log = Logger.get(TooManyOpenPartitionsFailureResolver.class); - private static final String NODE_RESOURCE_PATH = "/v1/node"; private final SqlParser sqlParser; private final PrestoAction prestoAction; private final Supplier testClusterSizeSupplier; private final int maxBucketPerWriter; + @Inject public TooManyOpenPartitionsFailureResolver( SqlParser sqlParser, PrestoAction prestoAction, @@ -72,7 +73,7 @@ public TooManyOpenPartitionsFailureResolver( } @Override - public Optional resolveQueryFailure(QueryStats controlQueryStats, QueryException queryException, Optional test) + public Optional resolveQueryFailure(QueryStatsEvent controlQueryStats, QueryException queryException, Optional test) { if (!test.isPresent()) { return Optional.empty(); @@ -92,7 +93,7 @@ public Optional resolveQueryFailure(QueryStats controlQueryStats, QueryE } long bucketCount = ((LongLiteral) getOnlyElement(bucketCountProperty).getValue()).getValue(); - int testClusterSize = testClusterSizeSupplier.get(); + int testClusterSize = this.testClusterSizeSupplier.get(); if (testClusterSize * maxBucketPerWriter < bucketCount) { return Optional.of("Not enough workers on test cluster"); @@ -109,12 +110,16 @@ public Optional resolveQueryFailure(QueryStats controlQueryStats, QueryE public static class Factory implements FailureResolverFactory { + private final ClusterSizeSupplier testClusterSizeSupplier; private final Duration clusterSizeExpiration; private final int maxBucketPerWriter; @Inject - public Factory(TooManyOpenPartitionsFailureResolverConfig config) + public Factory( + @ForTest ClusterSizeSupplier testClusterSizeSupplier, + TooManyOpenPartitionsFailureResolverConfig config) { + this.testClusterSizeSupplier = requireNonNull(testClusterSizeSupplier, "testClusterSizeSupplier is null"); this.clusterSizeExpiration = requireNonNull(config.getClusterSizeExpiration(), "clusterSizeExpiration is null"); this.maxBucketPerWriter = config.getMaxBucketsPerWriter(); } @@ -122,15 +127,13 @@ public Factory(TooManyOpenPartitionsFailureResolverConfig config) @Override public FailureResolver create(FailureResolverFactoryContext context) { - Supplier testClusterSizeSupplier = memoizeWithExpiration( - () -> context.getTestResourceClient().getClusterSize(NODE_RESOURCE_PATH), - clusterSizeExpiration.toMillis(), - MILLISECONDS); - return new TooManyOpenPartitionsFailureResolver( context.getSqlParser(), context.getPrestoAction(), - testClusterSizeSupplier, + memoizeWithExpiration( + testClusterSizeSupplier::getClusterSize, + clusterSizeExpiration.toMillis(), + MILLISECONDS), maxBucketPerWriter); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/VerifierLimitationFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/VerifierLimitationFailureResolver.java index 5926069770b62..1259d023db6e9 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/VerifierLimitationFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/VerifierLimitationFailureResolver.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.resolver; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; import com.google.common.collect.ImmutableSet; @@ -31,7 +31,7 @@ public class VerifierLimitationFailureResolver public static final String NAME = "verifier-limitation"; @Override - public Optional resolveQueryFailure(QueryStats controlQueryStats, QueryException queryException, Optional test) + public Optional resolveQueryFailure(QueryStatsEvent controlQueryStats, QueryException queryException, Optional test) { return mapMatchingPrestoException(queryException, CONTROL_CHECKSUM, ImmutableSet.of(COMPILER_ERROR, GENERATED_BYTECODE_TOO_LARGE), e -> Optional.of("Checksum query too large")); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/AbstractTestVerifierIntegrationSmokeTest.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/AbstractTestVerifierIntegrationSmokeTest.java index 50163811c6d2c..cd5b59e015cf9 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/AbstractTestVerifierIntegrationSmokeTest.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/AbstractTestVerifierIntegrationSmokeTest.java @@ -112,6 +112,7 @@ public void generateConfigFile() .put("control.jdbc-port", String.valueOf(port)) .put("test.host", host) .put("test.jdbc-port", String.valueOf(port)) + .put("test.http-port", String.valueOf(port)) .put("control.table-prefix", "local.tmp_verifier_c") .put("test.table-prefix", "local.tmp_verifier_t") .put("source-query.database", mySqlServer.getJdbcUrl(XDB)) diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java index 76126daddf91c..917e5512b91e8 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java @@ -25,8 +25,10 @@ import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; import com.facebook.presto.verifier.prestoaction.JdbcPrestoAction; import com.facebook.presto.verifier.prestoaction.PrestoAction; -import com.facebook.presto.verifier.prestoaction.PrestoClusterConfig; +import com.facebook.presto.verifier.prestoaction.PrestoActionConfig; import com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier; +import com.facebook.presto.verifier.prestoaction.QueryActions; +import com.facebook.presto.verifier.prestoaction.QueryActionsConfig; import com.facebook.presto.verifier.resolver.ChecksumExceededTimeLimitFailureResolver; import com.facebook.presto.verifier.resolver.ExceededGlobalMemoryLimitFailureResolver; import com.facebook.presto.verifier.resolver.ExceededTimeLimitFailureResolver; @@ -110,15 +112,18 @@ private Optional runVerification(SourceQuery sourceQuery, Op VerificationContext verificationContext = VerificationContext.create(); VerifierConfig verifierConfig = new VerifierConfig().setTestId(TEST_ID); RetryConfig retryConfig = new RetryConfig(); + QueryActionsConfig queryActionsConfig = new QueryActionsConfig(); TypeManager typeManager = createTypeManager(); PrestoAction prestoAction = mockPrestoAction.orElseGet(() -> { return new JdbcPrestoAction( exceptionClassifier, sourceQuery.getControlConfiguration(), verificationContext, - new PrestoClusterConfig() + new PrestoActionConfig() .setHost(queryRunner.getServer().getAddress().getHost()) .setJdbcPort(queryRunner.getServer().getAddress().getPort()), + queryActionsConfig.getMetadataTimeout(), + queryActionsConfig.getChecksumTimeout(), retryConfig, retryConfig); }); @@ -129,7 +134,7 @@ private Optional runVerification(SourceQuery sourceQuery, Op new QueryRewriteConfig().setTablePrefix("tmp_verifier_t")).create(prestoAction); ChecksumValidator checksumValidator = createChecksumValidator(verifierConfig); return new DataVerification( - prestoAction, + new QueryActions(prestoAction, prestoAction, prestoAction), sourceQuery, queryRewriter, new DeterminismAnalyzer( @@ -138,7 +143,6 @@ private Optional runVerification(SourceQuery sourceQuery, Op queryRewriter, checksumValidator, typeManager, - verificationContext, determinismAnalyzerConfig), new FailureResolverManager(ImmutableSet.of( new ExceededGlobalMemoryLimitFailureResolver(), @@ -473,5 +477,6 @@ private static void assertSuccessQueryInfo(QueryType queryType, QueryInfo queryI assertNotNull(queryInfo.getWallTimeSecs()); assertNotNull(queryInfo.getPeakTotalMemoryBytes()); assertNotNull(queryInfo.getPeakTaskTotalMemoryBytes()); + assertNotNull(queryInfo.getQueryStats()); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java index e95d6e1ffae48..d4efda01e9d81 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java @@ -21,8 +21,9 @@ import com.facebook.presto.verifier.checksum.ChecksumValidator; import com.facebook.presto.verifier.prestoaction.JdbcPrestoAction; import com.facebook.presto.verifier.prestoaction.PrestoAction; -import com.facebook.presto.verifier.prestoaction.PrestoClusterConfig; +import com.facebook.presto.verifier.prestoaction.PrestoActionConfig; import com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier; +import com.facebook.presto.verifier.prestoaction.QueryActionsConfig; import com.facebook.presto.verifier.retry.RetryConfig; import com.facebook.presto.verifier.rewrite.QueryRewriter; import com.google.common.collect.ImmutableMap; @@ -64,12 +65,15 @@ private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatal VerificationContext verificationContext = VerificationContext.create(); VerifierConfig verifierConfig = new VerifierConfig().setTestId("test-id"); RetryConfig retryConfig = new RetryConfig(); + QueryActionsConfig queryActionsConfig = new QueryActionsConfig(); TypeManager typeManager = createTypeManager(); PrestoAction prestoAction = new JdbcPrestoAction( PrestoExceptionClassifier.defaultBuilder().build(), configuration, verificationContext, - new PrestoClusterConfig(), + new PrestoActionConfig(), + queryActionsConfig.getMetadataTimeout(), + queryActionsConfig.getChecksumTimeout(), retryConfig, retryConfig); QueryRewriter queryRewriter = new QueryRewriter( @@ -87,7 +91,6 @@ private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatal queryRewriter, checksumValidator, typeManager, - verificationContext, new DeterminismAnalyzerConfig().setNonDeterministicCatalogs(mutableCatalogPattern)); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java index 995f4ab152fe4..e90b11c882732 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; @@ -21,6 +20,7 @@ import com.facebook.presto.verifier.TestingResultSetMetaData; import com.facebook.presto.verifier.TestingResultSetMetaData.ColumnInfo; import com.facebook.presto.verifier.event.DeterminismAnalysisDetails; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.prestoaction.PrestoAction; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -59,7 +59,7 @@ public MockPrestoAction(List rows, List columns) } @Override - public QueryStats execute(Statement statement, QueryStage queryStage) + public QueryStatsEvent execute(Statement statement, QueryStage queryStage) { throw new UnsupportedOperationException(); } @@ -79,7 +79,7 @@ public Statement getLastStatement() } private static final long ROW_COUNT_WITH_LIMIT = 1000; - private static final QueryStats QUERY_STATS = new QueryStats("id", "", false, false, 1, 2, 3, 4, 5, 0, 7, 8, 9, 10, 11, 0, 0, 0, Optional.empty()); + private static final QueryStatsEvent QUERY_STATS = new QueryStatsEvent("id", "", false, false, 1, 2, 3, 4, 5, 0, 7, 8, 9, 10, 11, 0, 0, 0, Optional.empty()); private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build(); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java index 3d09dcb6bf16e..ff54a6cd037ce 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java @@ -14,17 +14,17 @@ package com.facebook.presto.verifier.framework; import com.facebook.airlift.event.client.AbstractEventClient; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.type.TypeRegistry; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent; -import com.facebook.presto.verifier.prestoaction.NodeResourceClient; import com.facebook.presto.verifier.prestoaction.PrestoAction; import com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier; +import com.facebook.presto.verifier.prestoaction.QueryActions; import com.facebook.presto.verifier.resolver.FailureResolverManagerFactory; import com.facebook.presto.verifier.rewrite.QueryRewriter; import com.google.common.collect.ImmutableList; @@ -72,7 +72,7 @@ public MockPrestoAction(RuntimeException exception) } @Override - public QueryStats execute(Statement statement, QueryStage queryStage) + public QueryStatsEvent execute(Statement statement, QueryStage queryStage) { throw exceptionGenerator.apply(queryStage); } @@ -87,16 +87,6 @@ public QueryResult execute( } } - private static class MockNodeResourceClient - implements NodeResourceClient - { - @Override - public int getClusterSize(String path) - { - throw new UnsupportedOperationException(); - } - } - private static class MockEventClient extends AbstractEventClient { @@ -217,10 +207,9 @@ private VerificationManager getVerificationManager(List sourceQueri () -> sourceQueries, new VerificationFactory( SQL_PARSER, - (sourceQuery, verificationContext) -> prestoAction, + (sourceQuery, verificationContext) -> new QueryActions(prestoAction, prestoAction, prestoAction), presto -> new QueryRewriter(SQL_PARSER, createTypeManager(), presto, ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX), ImmutableMap.of()), new FailureResolverManagerFactory(ImmutableSet.of(), ImmutableSet.of()), - new MockNodeResourceClient(), createChecksumValidator(verifierConfig), PrestoExceptionClassifier.defaultBuilder().build(), verifierConfig, diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java index 77b97556a22b7..2c2154f8e6104 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java @@ -41,7 +41,9 @@ public void testDefault() .setRelativeErrorMargin(1e-4) .setAbsoluteErrorMargin(1e-12) .setSmartTeardown(false) - .setVerificationResubmissionLimit(2)); + .setVerificationResubmissionLimit(2) + .setSetupOnMainClusters(true) + .setTeardownOnMainClusters(true)); } @Test @@ -62,6 +64,8 @@ public void testExplicitPropertyMappings() .put("absolute-error-margin", "1e-14") .put("smart-teardown", "true") .put("verification-resubmission.limit", "1") + .put("setup-on-main-clusters", "false") + .put("teardown-on-main-clusters", "false") .build(); VerifierConfig expected = new VerifierConfig() .setWhitelist("a,b,c") @@ -77,7 +81,9 @@ public void testExplicitPropertyMappings() .setRelativeErrorMargin(2e-5) .setAbsoluteErrorMargin(1e-14) .setSmartTeardown(true) - .setVerificationResubmissionLimit(1); + .setVerificationResubmissionLimit(1) + .setSetupOnMainClusters(false) + .setTeardownOnMainClusters(false); assertFullMapping(properties, expected); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java index 07017493ebb79..ee1820dda680b 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java @@ -71,14 +71,17 @@ public void setupQueryRunner() @BeforeMethod public void setup() { + QueryActionsConfig queryActionsConfig = new QueryActionsConfig(); verificationContext = VerificationContext.create(); prestoAction = new JdbcPrestoAction( PrestoExceptionClassifier.defaultBuilder().build(), CONFIGURATION, verificationContext, - new PrestoClusterConfig() + new PrestoActionConfig() .setHost(queryRunner.getServer().getAddress().getHost()) .setJdbcPort(queryRunner.getServer().getAddress().getPort()), + queryActionsConfig.getMetadataTimeout(), + queryActionsConfig.getChecksumTimeout(), new RetryConfig(), new RetryConfig()); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoClusterConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoActionConfig.java similarity index 78% rename from presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoClusterConfig.java rename to presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoActionConfig.java index 7ace6fc84542b..46f3d59425d5e 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoClusterConfig.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoActionConfig.java @@ -27,19 +27,17 @@ import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; -public class TestPrestoClusterConfig +public class TestPrestoActionConfig { @Test public void testDefault() { - assertRecordedDefaults(recordDefaults(PrestoClusterConfig.class) + assertRecordedDefaults(recordDefaults(PrestoActionConfig.class) .setHost(null) .setJdbcPort(0) - .setHttpPort(0) + .setHttpPort(null) .setJdbcUrlParameters(null) - .setQueryTimeout(new Duration(60, MINUTES)) - .setMetadataTimeout(new Duration(3, MINUTES)) - .setChecksumTimeout(new Duration(30, MINUTES))); + .setQueryTimeout(new Duration(60, MINUTES))); } @Test @@ -51,17 +49,13 @@ public void testExplicitPropertyMappings() .put("http-port", "7777") .put("jdbc-url-parameters", "{\"SSL\": false}") .put("query-timeout", "2h") - .put("metadata-timeout", "1h") - .put("checksum-timeout", "3h") .build(); - PrestoClusterConfig expected = new PrestoClusterConfig() + PrestoActionConfig expected = new PrestoActionConfig() .setHost("proxy.presto.fbinfra.net") .setJdbcPort(7778) .setHttpPort(7777) .setJdbcUrlParameters("{\"SSL\": false}") - .setQueryTimeout(new Duration(2, HOURS)) - .setMetadataTimeout(new Duration(1, HOURS)) - .setChecksumTimeout(new Duration(3, HOURS)); + .setQueryTimeout(new Duration(2, HOURS)); assertFullMapping(properties, expected); } @@ -69,7 +63,7 @@ public void testExplicitPropertyMappings() @Test public void testJdbcUrl() { - PrestoClusterConfig config = new PrestoClusterConfig() + PrestoActionConfig config = new PrestoActionConfig() .setHost("proxy.presto.fbinfra.net") .setJdbcPort(7778) .setJdbcUrlParameters("{\"SSL\": true, \"SSLTrustStorePath\": \"trust-store\", \"SSLKeyStorePath\": \"key-store\"}") @@ -80,7 +74,7 @@ public void testJdbcUrl() @Test public void testHttpUri() { - PrestoClusterConfig config = new PrestoClusterConfig() + PrestoActionConfig config = new PrestoActionConfig() .setHost("proxy.presto.fbinfra.net") .setJdbcPort(7778) .setHttpPort(7777) diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoExceptionClassifier.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoExceptionClassifier.java index 0fb3c52beb728..8e9f4561bafb2 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoExceptionClassifier.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoExceptionClassifier.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.verifier.prestoaction; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.ClusterConnectionException; import com.facebook.presto.verifier.framework.PrestoQueryException; import com.facebook.presto.verifier.framework.QueryException; @@ -55,7 +55,7 @@ public class TestPrestoExceptionClassifier { private static final QueryStage QUERY_STAGE = CONTROL_MAIN; - private static final QueryStats QUERY_STATS = new QueryStats("id", "", false, false, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, Optional.empty()); + private static final QueryStatsEvent QUERY_STATS = new QueryStatsEvent("id", "", false, false, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, Optional.empty()); private final SqlExceptionClassifier classifier = PrestoExceptionClassifier.defaultBuilder().build(); @@ -146,7 +146,7 @@ private void assertPrestoQueryException( QueryException queryException, Optional errorCode, boolean retryable, - Optional queryStats, + Optional queryStats, QueryStage queryStage) { assertTrue(queryException instanceof PrestoQueryException); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestQueryActionsConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestQueryActionsConfig.java new file mode 100644 index 0000000000000..dad2c1587a5a7 --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestQueryActionsConfig.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MINUTES; + +public class TestQueryActionsConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(QueryActionsConfig.class) + .setControlQueryActionType(JdbcPrestoAction.QUERY_ACTION_TYPE) + .setTestQueryActionType(JdbcPrestoAction.QUERY_ACTION_TYPE) + .setRunHelperQueriesOnControl(true) + .setMetadataTimeout(new Duration(3, MINUTES)) + .setChecksumTimeout(new Duration(30, MINUTES))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("control.query-action-type", "control-action") + .put("test.query-action-type", "test-action") + .put("run-helper-queries-on-control", "false") + .put("metadata-timeout", "1h") + .put("checksum-timeout", "3h") + .build(); + QueryActionsConfig expected = new QueryActionsConfig() + .setControlQueryActionType("control-action") + .setTestQueryActionType("test-action") + .setRunHelperQueriesOnControl(false) + .setMetadataTimeout(new Duration(1, HOURS)) + .setChecksumTimeout(new Duration(3, HOURS)); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/AbstractTestPrestoQueryFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/AbstractTestPrestoQueryFailureResolver.java index ce4d75b4961cd..58a54ed30c370 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/AbstractTestPrestoQueryFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/AbstractTestPrestoQueryFailureResolver.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.resolver; -import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.ClusterConnectionException; import com.facebook.presto.verifier.framework.PrestoQueryException; import org.testng.annotations.Test; @@ -32,7 +32,7 @@ public class AbstractTestPrestoQueryFailureResolver { protected static final long CONTROL_CPU_TIME_MILLIS = 100000; protected static final long CONTROL_PEAK_TOTAL_MEMORY_BYTES = 600L * 1024 * 1024 * 1024; - protected static final QueryStats CONTROL_QUERY_STATS = createQueryStats(CONTROL_CPU_TIME_MILLIS, CONTROL_PEAK_TOTAL_MEMORY_BYTES); + protected static final QueryStatsEvent CONTROL_QUERY_STATS = createQueryStats(CONTROL_CPU_TIME_MILLIS, CONTROL_PEAK_TOTAL_MEMORY_BYTES); private final FailureResolver failureResolver; @@ -101,8 +101,8 @@ protected FailureResolver getFailureResolver() return failureResolver; } - protected static QueryStats createQueryStats(long cpuTimeMillls, long peakTotalMemoryBytes) + protected static QueryStatsEvent createQueryStats(long cpuTimeMillls, long peakTotalMemoryBytes) { - return new QueryStats("id", "", false, false, 1, 2, 3, 4, 5, cpuTimeMillls, 7, 8, 9, 10, 11, 12, peakTotalMemoryBytes, 13, Optional.empty()); + return new QueryStatsEvent("id", "", false, false, 1, 2, 3, 4, 5, cpuTimeMillls, 7, 8, 9, 10, 11, 12, peakTotalMemoryBytes, 13, Optional.empty()); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestHttpNodeResourceClient.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestClusterSizeFetcher.java similarity index 74% rename from presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestHttpNodeResourceClient.java rename to presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestClusterSizeFetcher.java index 31e2dd0b1aea4..f5182f14d8f01 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestHttpNodeResourceClient.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestClusterSizeFetcher.java @@ -11,25 +11,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.prestoaction; +package com.facebook.presto.verifier.resolver; import com.facebook.airlift.http.client.jetty.JettyHttpClient; import com.facebook.presto.server.testing.TestingPrestoServer; +import com.facebook.presto.verifier.prestoaction.PrestoActionConfig; import com.facebook.presto.verifier.retry.RetryConfig; import org.testng.annotations.Test; -public class TestHttpNodeResourceClient +import static org.testng.Assert.assertEquals; + +public class TestClusterSizeFetcher { private final TestingPrestoServer server; - private final HttpNodeResourceClient client; + private final ClusterSizeFetcher client; - public TestHttpNodeResourceClient() + public TestClusterSizeFetcher() throws Exception { server = new TestingPrestoServer(); - client = new HttpNodeResourceClient( + client = new ClusterSizeFetcher( new JettyHttpClient(), - new PrestoClusterConfig() + new PrestoActionConfig() .setHost(server.getAddress().getHost()) .setHttpPort(server.getAddress().getPort()), new RetryConfig()); @@ -38,6 +41,6 @@ public TestHttpNodeResourceClient() @Test public void testNodeResource() { - client.getClusterSize("/v1/node"); + assertEquals(client.getClusterSize(), 0); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java index 6c5019d076eca..af2fd21ca9173 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.verifier.resolver; -import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.parser.IdentifierSymbol; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; @@ -22,6 +21,7 @@ import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.verifier.TestingResultSetMetaData; import com.facebook.presto.verifier.TestingResultSetMetaData.ColumnInfo; +import com.facebook.presto.verifier.event.QueryStatsEvent; import com.facebook.presto.verifier.framework.PrestoQueryException; import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; @@ -62,7 +62,7 @@ public MockPrestoAction(AtomicReference createTable) } @Override - public QueryStats execute(Statement statement, QueryStage queryStage) + public QueryStatsEvent execute(Statement statement, QueryStage queryStage) { throw new UnsupportedOperationException(); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java index ba9f82c437bbd..2a668235e6d2e 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java @@ -26,8 +26,9 @@ import com.facebook.presto.verifier.framework.VerificationContext; import com.facebook.presto.verifier.prestoaction.JdbcPrestoAction; import com.facebook.presto.verifier.prestoaction.PrestoAction; -import com.facebook.presto.verifier.prestoaction.PrestoClusterConfig; +import com.facebook.presto.verifier.prestoaction.PrestoActionConfig; import com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier; +import com.facebook.presto.verifier.prestoaction.QueryActionsConfig; import com.facebook.presto.verifier.retry.RetryConfig; import com.google.common.collect.ImmutableList; import org.intellij.lang.annotations.Language; @@ -75,9 +76,11 @@ public void setup() PrestoExceptionClassifier.defaultBuilder().build(), CONFIGURATION, VerificationContext.create(), - new PrestoClusterConfig() + new PrestoActionConfig() .setHost(queryRunner.getServer().getAddress().getHost()) .setJdbcPort(queryRunner.getServer().getAddress().getPort()), + new QueryActionsConfig().getMetadataTimeout(), + new QueryActionsConfig().getChecksumTimeout(), new RetryConfig(), new RetryConfig()); }