diff --git a/presto-docs/src/main/sphinx/admin/verifier.rst b/presto-docs/src/main/sphinx/admin/verifier.rst index cd99f117cc3aa..57007ae8fdfa8 100644 --- a/presto-docs/src/main/sphinx/admin/verifier.rst +++ b/presto-docs/src/main/sphinx/admin/verifier.rst @@ -276,6 +276,10 @@ Name Description are emitted to ``stdout``. ``control.table-prefix`` The table prefix to be appended to the control target table. ``test.table-prefix`` The table prefix to be appended to the test target table. +``control.reuse-table`` If ``true``, reuse the output table of the control source Insert and CreateTableAsSelect + query. Otherwise, run the control source query and write to a temporary table. +``test.reuse-table`` If ``true``, reuse the output table of the test source Insert and CreateTableAsSelect + query. Otherwise, run the test source query and write to a temporary table. ``test-id`` A string to be attached to output events. ``max-concurrency`` Maximum number of concurrent verifications. ``suite-repetition`` How many times a suite is verified. diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java index 41143023efd04..6929cca10cbbb 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java @@ -53,12 +53,11 @@ public class QueryInfo private final String bucketChecksumQuery; private final String jsonPlan; private final String outputTableName; - + private final boolean isReuseTable; private final Double cpuTimeSecs; private final Double wallTimeSecs; private final Long peakTotalMemoryBytes; private final Long peakTaskTotalMemoryBytes; - private final String extraStats; public QueryInfo( @@ -79,7 +78,8 @@ public QueryInfo( Optional bucketChecksumQuery, Optional jsonPlan, Optional queryActionStats, - Optional outputTableName) + Optional outputTableName, + boolean isReuseTable) { Optional stats = queryActionStats.flatMap(QueryActionStats::getQueryStats); this.catalog = requireNonNull(catalog, "catalog is null"); @@ -105,6 +105,7 @@ public QueryInfo( this.peakTaskTotalMemoryBytes = stats.map(QueryStats::getPeakTaskTotalMemoryBytes).orElse(null); this.extraStats = queryActionStats.flatMap(QueryActionStats::getExtraStats).orElse(null); this.outputTableName = outputTableName.orElse(null); + this.isReuseTable = isReuseTable; } private static double millisToSeconds(long millis) @@ -220,6 +221,12 @@ public String getOutputTableName() return outputTableName; } + @EventField + public boolean getIsReuseTable() + { + return isReuseTable; + } + @EventField public Double getCpuTimeSecs() { @@ -279,6 +286,7 @@ public static class Builder private Optional jsonPlan = Optional.empty(); private Optional queryActionStats = Optional.empty(); private Optional outputTableName = Optional.empty(); + private boolean isReuseTable; private Builder( String catalog, @@ -376,6 +384,12 @@ public Builder setOutputTableName(Optional outputTableName) return this; } + public Builder setIsReuseTable(boolean isReuseTable) + { + this.isReuseTable = isReuseTable; + return this; + } + public QueryInfo build() { return new QueryInfo( @@ -396,7 +410,8 @@ public QueryInfo build() bucketChecksumQuery, jsonPlan, queryActionStats, - outputTableName); + outputTableName, + isReuseTable); } } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java index 1a7f938a15d22..ef27432a7d725 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java @@ -39,6 +39,7 @@ public enum EventStatus FAILED, FAILED_RESOLVED, SKIPPED, + RESUBMITTED, } private final String suite; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java index d44f7a1462725..cd94ba3be540b 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java @@ -196,6 +196,8 @@ public VerificationResult run() ChecksumQueryContext testChecksumQueryContext = new ChecksumQueryContext(); Optional matchResult = Optional.empty(); Optional determinismAnalysisDetails = Optional.empty(); + boolean controlReuseTable = false; + boolean testReuseTable = false; Optional partialResult = Optional.empty(); Optional throwable = Optional.empty(); @@ -204,11 +206,22 @@ public VerificationResult run() // Rewrite queries if (isControlEnabled()) { control = Optional.of(getQueryRewrite(CONTROL)); + controlReuseTable = control.isPresent() && control.get() instanceof QueryObjectBundle && ((QueryObjectBundle) control.get()).isReuseTable(); + if (controlReuseTable) { + controlQueryContext.setState(QueryState.REUSE); + controlQueryContext.setMainQueryStats(QueryActionStats.queryIdStats(sourceQuery.getQueryId(CONTROL).get())); + } } + test = Optional.of(getQueryRewrite(TEST)); + testReuseTable = test.isPresent() && test.get() instanceof QueryObjectBundle && ((QueryObjectBundle) test.get()).isReuseTable(); + if (testReuseTable) { + testQueryContext.setState(QueryState.REUSE); + testQueryContext.setMainQueryStats(QueryActionStats.queryIdStats(sourceQuery.getQueryId(TEST).get())); + } // First run setup queries - if (isControlEnabled()) { + if (isControlEnabled() && !controlReuseTable) { QueryBundle controlQueryBundle = control.get(); QueryAction controlSetupAction = setupOnMainClusters ? queryActions.getControlAction() : queryActions.getHelperAction(); controlQueryBundle.getSetupQueries().forEach(query -> runAndConsume( @@ -216,40 +229,51 @@ public VerificationResult run() controlQueryContext::addSetupQuery, controlQueryContext::setException)); } - QueryBundle testQueryBundle = test.get(); - QueryAction testSetupAction = setupOnMainClusters ? queryActions.getTestAction() : queryActions.getHelperAction(); - testQueryBundle.getSetupQueries().forEach(query -> runAndConsume( - () -> testSetupAction.execute(query, TEST_SETUP), - testQueryContext::addSetupQuery, - testQueryContext::setException)); + if (!testReuseTable) { + QueryBundle testQueryBundle = test.get(); + QueryAction testSetupAction = setupOnMainClusters ? queryActions.getTestAction() : queryActions.getHelperAction(); + testQueryBundle.getSetupQueries().forEach(query -> runAndConsume( + () -> testSetupAction.execute(query, TEST_SETUP), + testQueryContext::addSetupQuery, + testQueryContext::setException)); + } ListenableFuture>> controlQueryFuture = immediateFuture(Optional.empty()); + ListenableFuture>> testQueryFuture = immediateFuture(Optional.empty()); // Start control query - if (isControlEnabled()) { + if (isControlEnabled() && !controlReuseTable) { QueryBundle controlQueryBundle = control.get(); controlQueryFuture = executor.submit(() -> runMainQuery(controlQueryBundle.getQuery(), CONTROL, controlQueryContext)); } - if (!concurrentControlAndTest) { getFutureValue(controlQueryFuture); } // Run test queries - ListenableFuture>> testQueryFuture = executor.submit(() -> runMainQuery(testQueryBundle.getQuery(), TEST, testQueryContext)); + if (!testReuseTable) { + QueryBundle testQueryBundle = test.get(); + testQueryFuture = executor.submit(() -> runMainQuery(testQueryBundle.getQuery(), TEST, testQueryContext)); + } controlQueryResult = getFutureValue(controlQueryFuture); + if (QUERY_BANK_MODE.equals(runningMode) && !saveSnapshot) { controlQueryContext.setState(QueryState.SUCCEEDED); controlQueryContext.setMainQueryStats(EMPTY_STATS); } else if (!skipControl || QUERY_BANK_MODE.equals(runningMode)) { // saveSnapshot or regular run with skipControl = false - controlQueryContext.setState(QueryState.SUCCEEDED); + if (!controlReuseTable) { + controlQueryContext.setState(QueryState.SUCCEEDED); + } } else { controlQueryContext.setState(NOT_RUN); } + testQueryResult = getFutureValue(testQueryFuture); - testQueryContext.setState(QueryState.SUCCEEDED); + if (!testReuseTable) { + testQueryContext.setState(QueryState.SUCCEEDED); + } // Verify results if (QUERY_BANK_MODE.equals(runningMode) && !saveSnapshot && !skipChecksum) { @@ -262,8 +286,13 @@ else if ((isControlEnabled()) && !skipChecksum) { matchResult = Optional.of(verify(control.get(), test.get(), controlQueryResult, testQueryResult, controlChecksumQueryContext, testChecksumQueryContext)); // Determinism analysis - if (!QUERY_BANK_MODE.equals(runningMode) && matchResult.get().isMismatchPossiblyCausedByNonDeterminism()) { - determinismAnalysisDetails = Optional.of(analyzeDeterminism(control.get(), matchResult.get())); + if (!QUERY_BANK_MODE.equals(runningMode)) { + if ((controlReuseTable || testReuseTable) && matchResult.get().isMismatchPossiblyCausedByReuseOutdatedTable() && verificationContext.getResubmissionCount() < verificationResubmissionLimit) { + return new VerificationResult(this, true, Optional.empty()); + } + else if (matchResult.get().isMismatchPossiblyCausedByNonDeterminism()) { + determinismAnalysisDetails = Optional.of(analyzeDeterminism(control.get(), matchResult.get())); + } } } @@ -287,7 +316,7 @@ else if ((isControlEnabled()) && !skipChecksum) { } finally { if (!smartTeardown - || testQueryContext.getState() != QueryState.SUCCEEDED + || !ImmutableList.of(QueryState.SUCCEEDED, QueryState.REUSE).contains(testQueryContext.getState()) || (partialResult.isPresent() && partialResult.get().getStatus().equals(SUCCEEDED))) { QueryAction controlTeardownAction = teardownOnMainClusters ? queryActions.getControlAction() : queryActions.getHelperAction(); QueryAction testTeardownAction = teardownOnMainClusters ? queryActions.getTestAction() : queryActions.getHelperAction(); @@ -347,13 +376,14 @@ private EventStatus getEventStatus( } if (skipControl) { - if (testQueryContext.getState() == QueryState.SUCCEEDED) { + if (ImmutableList.of(QueryState.SUCCEEDED, QueryState.REUSE).contains(testQueryContext.getState())) { return SUCCEEDED; } } else { if (skipChecksum) { - if (controlQueryContext.getState() == QueryState.SUCCEEDED && testQueryContext.getState() == QueryState.SUCCEEDED) { + if (ImmutableList.of(QueryState.SUCCEEDED, QueryState.REUSE).contains(testQueryContext.getState()) && + ImmutableList.of(QueryState.SUCCEEDED, QueryState.REUSE).contains(controlQueryContext.getState())) { return SUCCEEDED; } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateTableVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateTableVerification.java index 87dd61a8ad3ff..c0eaa59418847 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateTableVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateTableVerification.java @@ -67,7 +67,7 @@ public CreateTableVerification( @Override protected QueryObjectBundle getQueryRewrite(ClusterType clusterType) { - return queryRewriter.rewriteQuery(getSourceQuery().getQuery(clusterType), clusterType); + return queryRewriter.rewriteQuery(getSourceQuery().getQuery(clusterType), getSourceQuery().getQueryConfiguration(clusterType), clusterType); } @Override diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateViewVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateViewVerification.java index 3b81975529e1c..18fed093f647d 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateViewVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateViewVerification.java @@ -67,7 +67,7 @@ public CreateViewVerification( @Override protected QueryObjectBundle getQueryRewrite(ClusterType clusterType) { - return queryRewriter.rewriteQuery(getSourceQuery().getQuery(clusterType), clusterType); + return queryRewriter.rewriteQuery(getSourceQuery().getQuery(clusterType), getSourceQuery().getQueryConfiguration(clusterType), clusterType); } @Override diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataMatchResult.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataMatchResult.java index 542005a702a73..6de343581713a 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataMatchResult.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataMatchResult.java @@ -97,6 +97,12 @@ public boolean isMismatchPossiblyCausedByNonDeterminism() return matchType == ROW_COUNT_MISMATCH || matchType == COLUMN_MISMATCH; } + @Override + public boolean isMismatchPossiblyCausedByReuseOutdatedTable() + { + return matchType == SCHEMA_MISMATCH || matchType == ROW_COUNT_MISMATCH || matchType == COLUMN_MISMATCH; + } + public MatchType getMatchType() { return matchType; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java index d137b3d4173b9..1e9cf1b4e12d9 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java @@ -85,15 +85,17 @@ public DataVerification( @Override protected QueryObjectBundle getQueryRewrite(ClusterType clusterType) { - return queryRewriter.rewriteQuery(getSourceQuery().getQuery(clusterType), clusterType); + return queryRewriter.rewriteQuery(getSourceQuery().getQuery(clusterType), getSourceQuery().getQueryConfiguration(clusterType), clusterType, + getVerificationContext().getResubmissionCount() == 0); } @Override protected void updateQueryInfoWithQueryBundle(QueryInfo.Builder queryInfo, Optional queryBundle) { super.updateQueryInfoWithQueryBundle(queryInfo, queryBundle); - queryInfo.setQuery(queryBundle.map(bundle -> formatSql(bundle.getQuery(), bundle.getRewrittenFunctionCalls()))); - queryInfo.setOutputTableName(queryBundle.map(QueryObjectBundle::getObjectName).map(QualifiedName::toString)); + queryInfo.setQuery(queryBundle.map(bundle -> formatSql(bundle.getQuery(), bundle.getRewrittenFunctionCalls()))) + .setOutputTableName(queryBundle.map(QueryObjectBundle::getObjectName).map(QualifiedName::toString)) + .setIsReuseTable(queryBundle.map(QueryObjectBundle::isReuseTable).orElse(false)); } @Override @@ -174,7 +176,7 @@ protected Optional resolveFailure( checkState(control.isPresent(), "control is missing"); return failureResolverManager.resolveResultMismatch((DataMatchResult) matchResult.get(), control.get()); } - if (throwable.isPresent() && controlQueryContext.getState() == QueryState.SUCCEEDED) { + if (throwable.isPresent() && ImmutableList.of(QueryState.SUCCEEDED, QueryState.REUSE).contains(controlQueryContext.getState())) { checkState(controlQueryContext.getMainQueryStats().isPresent(), "controlQueryStats is missing"); return failureResolverManager.resolveException(controlQueryContext.getMainQueryStats().get(), throwable.get(), test); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DdlMatchResult.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DdlMatchResult.java index 6a5ad4f1a2d56..0fb682656cecc 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DdlMatchResult.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DdlMatchResult.java @@ -70,6 +70,12 @@ public boolean isMismatchPossiblyCausedByNonDeterminism() return false; } + @Override + public boolean isMismatchPossiblyCausedByReuseOutdatedTable() + { + return false; + } + @Override public String getReport() { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java index 63bdd3a206b25..3283aaa367be5 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java @@ -131,7 +131,7 @@ private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult co Map queryRuns = new HashMap<>(); try { for (int i = 0; i < maxAnalysisRuns; i++) { - QueryObjectBundle queryBundle = queryRewriter.rewriteQuery(sourceQuery.getQuery(CONTROL), CONTROL); + QueryObjectBundle queryBundle = queryRewriter.rewriteQuery(sourceQuery.getQuery(CONTROL), sourceQuery.getQueryConfiguration(CONTROL), CONTROL, false); DeterminismAnalysisRun.Builder run = determinismAnalysisDetails.addRun().setTableName(queryBundle.getObjectName().toString()); queryRuns.put(queryBundle, run); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExplainMatchResult.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExplainMatchResult.java index 373bf70db5b99..923db4eade3e8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExplainMatchResult.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExplainMatchResult.java @@ -57,6 +57,12 @@ public boolean isMismatchPossiblyCausedByNonDeterminism() return false; } + @Override + public boolean isMismatchPossiblyCausedByReuseOutdatedTable() + { + return false; + } + @Override public String getReport() { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/MatchResult.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/MatchResult.java index 40a99bf755b5c..ca2e5f804d95b 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/MatchResult.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/MatchResult.java @@ -23,5 +23,7 @@ public interface MatchResult boolean isMismatchPossiblyCausedByNonDeterminism(); + boolean isMismatchPossiblyCausedByReuseOutdatedTable(); + String getReport(); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java index 64a57379dec06..3ae5f0e2d7718 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java @@ -18,6 +18,7 @@ import org.jdbi.v3.core.mapper.reflect.JdbiConstructor; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -29,11 +30,13 @@ public class QueryConfiguration { + public static final String CLIENT_TAG_OUTPUT_RETAINED = "OUTPUT_RETAINED"; private final String catalog; private final String schema; private final Optional username; private final Optional password; private final Map sessionProperties; + private final boolean isReusableTable; @JdbiConstructor public QueryConfiguration( @@ -41,13 +44,26 @@ public QueryConfiguration( @ColumnName("schema") String schema, @ColumnName("username") Optional username, @ColumnName("password") Optional password, - @ColumnName("session_properties") Optional> sessionProperties) + @ColumnName("session_properties") Optional> sessionProperties, + @ColumnName("client_tags") Optional> clientTags) + { + this(catalog, schema, username, password, sessionProperties, clientTags.filter(tags -> tags.contains(CLIENT_TAG_OUTPUT_RETAINED)).isPresent()); + } + + public QueryConfiguration( + String catalog, + String schema, + Optional username, + Optional password, + Optional> sessionProperties, + boolean isReusableTable) { this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); this.username = requireNonNull(username, "username is null"); this.password = requireNonNull(password, "password is null"); this.sessionProperties = ImmutableMap.copyOf(sessionProperties.orElse(ImmutableMap.of())); + this.isReusableTable = isReusableTable; } public QueryConfiguration applyOverrides(QueryConfigurationOverrides overrides) @@ -68,7 +84,8 @@ public QueryConfiguration applyOverrides(QueryConfigurationOverrides overrides) overrides.getSchemaOverride().orElse(schema), Optional.ofNullable(overrides.getUsernameOverride().orElse(username.orElse(null))), Optional.ofNullable(overrides.getPasswordOverride().orElse(password.orElse(null))), - Optional.of(sessionProperties)); + Optional.of(sessionProperties), + isReusableTable); } public String getCatalog() @@ -96,6 +113,11 @@ public Map getSessionProperties() return sessionProperties; } + public boolean isReusableTable() + { + return isReusableTable; + } + @Override public boolean equals(Object obj) { @@ -110,13 +132,14 @@ public boolean equals(Object obj) Objects.equals(schema, o.schema) && Objects.equals(username, o.username) && Objects.equals(password, o.password) && - Objects.equals(sessionProperties, o.sessionProperties); + Objects.equals(sessionProperties, o.sessionProperties) && + isReusableTable == o.isReusableTable; } @Override public int hashCode() { - return Objects.hash(catalog, schema, username, password, sessionProperties); + return Objects.hash(catalog, schema, username, password, sessionProperties, isReusableTable); } @Override @@ -128,6 +151,7 @@ public String toString() .add("username", username) .add("password", password) .add("sessionProperties", sessionProperties) + .add("isReusableTable", isReusableTable) .toString(); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java index c670f58a2ac31..0465157956e2c 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java @@ -25,6 +25,7 @@ public class QueryObjectBundle extends QueryBundle { private final QualifiedName objectName; + private final boolean reuseTable; private final Optional rewrittenFunctionCalls; public QueryObjectBundle( @@ -33,11 +34,13 @@ public QueryObjectBundle( Statement query, List teardownQueries, ClusterType cluster, - Optional rewrittenFunctionCalls) + Optional rewrittenFunctionCalls, + boolean reuseTable) { super(setupQueries, query, teardownQueries, cluster); this.objectName = requireNonNull(objectName, "objectName is null"); this.rewrittenFunctionCalls = requireNonNull(rewrittenFunctionCalls, "rewrittenFunctionCalls is null"); + this.reuseTable = reuseTable; } public QualifiedName getObjectName() @@ -49,4 +52,9 @@ public Optional getRewrittenFunctionCalls() { return rewrittenFunctionCalls; } + + public boolean isReuseTable() + { + return reuseTable; + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryState.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryState.java index f5536478b3d61..f0e4987da34ad 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryState.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryState.java @@ -16,6 +16,7 @@ public enum QueryState { SUCCEEDED, + REUSE, FAILED, TIMED_OUT, FAILED_TO_SETUP, diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SourceQuery.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SourceQuery.java index 8878c17f13104..9f99ce2090b55 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SourceQuery.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SourceQuery.java @@ -18,6 +18,7 @@ import org.jdbi.v3.core.mapper.reflect.JdbiConstructor; import java.util.Objects; +import java.util.Optional; import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; import static com.facebook.presto.verifier.framework.ClusterType.TEST; @@ -31,6 +32,8 @@ public class SourceQuery private final String name; private final String controlQuery; private final String testQuery; + private final Optional controlQueryId; + private final Optional testQueryId; private final QueryConfiguration controlConfiguration; private final QueryConfiguration testConfiguration; @@ -40,6 +43,8 @@ public SourceQuery( @ColumnName("name") String name, @ColumnName("controlQuery") String controlQuery, @ColumnName("testQuery") String testQuery, + @ColumnName("controlQueryId") Optional controlQueryId, + @ColumnName("testQueryId") Optional testQueryId, @Nested("control") QueryConfiguration controlConfiguration, @Nested("test") QueryConfiguration testConfiguration) { @@ -47,6 +52,8 @@ public SourceQuery( this.name = requireNonNull(name, "name is null"); this.controlQuery = clean(controlQuery); this.testQuery = clean(testQuery); + this.controlQueryId = requireNonNull(controlQueryId, "controlQueryId is null"); + this.testQueryId = requireNonNull(testQueryId, "testQueryId is null"); this.controlConfiguration = requireNonNull(controlConfiguration, "controlConfiguration is null"); this.testConfiguration = requireNonNull(testConfiguration, "testConfiguration is null"); } @@ -67,6 +74,18 @@ public String getQuery(ClusterType clusterType) return clusterType == CONTROL ? controlQuery : testQuery; } + public Optional getQueryId(ClusterType clusterType) + { + checkArgument(clusterType == CONTROL || clusterType == TEST, "Invalid ClusterType: %s", clusterType); + return clusterType == CONTROL ? controlQueryId : testQueryId; + } + + public QueryConfiguration getQueryConfiguration(ClusterType clusterType) + { + checkArgument(clusterType == CONTROL || clusterType == TEST, "Invalid ClusterType: %s", clusterType); + return clusterType == CONTROL ? controlConfiguration : testConfiguration; + } + public QueryConfiguration getControlConfiguration() { return controlConfiguration; @@ -102,6 +121,8 @@ public boolean equals(Object obj) Objects.equals(name, o.name) && Objects.equals(controlQuery, o.controlQuery) && Objects.equals(testQuery, o.testQuery) && + Objects.equals(controlQueryId, o.controlQueryId) && + Objects.equals(testQueryId, o.testQueryId) && Objects.equals(controlConfiguration, o.controlConfiguration) && Objects.equals(testConfiguration, o.testConfiguration); } @@ -109,7 +130,7 @@ public boolean equals(Object obj) @Override public int hashCode() { - return Objects.hash(suite, name, controlQuery, testQuery, controlConfiguration, testConfiguration); + return Objects.hash(suite, name, controlQuery, testQuery, controlQueryId, testQueryId, controlConfiguration, testConfiguration); } @Override @@ -120,6 +141,8 @@ public String toString() .add("name", name) .add("controlQuery", controlQuery) .add("testQuery", testQuery) + .add("controlQueryId", controlQueryId) + .add("testQueryId", testQueryId) .add("controlConfiguration", controlConfiguration) .add("testConfiguration", testConfiguration) .toString(); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationManager.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationManager.java index e451dbe97757d..8d4bd1806e87b 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationManager.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationManager.java @@ -70,6 +70,7 @@ import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.FAILED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.FAILED_RESOLVED; +import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.RESUBMITTED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SUCCEEDED; import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; @@ -204,7 +205,7 @@ private void resubmit(Verification verification) Verification newVerification = verificationFactory.get(sourceQuery, Optional.of(newContext), snapshotQueryConsumer, snapshotQueries); completionService.submit(newVerification::run); queriesSubmitted.addAndGet(1); - log.info("Verification %s failed, resubmitted for verification (%s/%s)", sourceQuery.getName(), newContext.getResubmissionCount(), verificationResubmissionLimit); + log.info("Resubmitted %s for verification (%s/%s)", sourceQuery.getName(), newContext.getResubmissionCount(), verificationResubmissionLimit); } @VisibleForTesting @@ -221,6 +222,8 @@ private List applyOverrides(List sourceQueries) sourceQuery.getName(), sourceQuery.getQuery(CONTROL), sourceQuery.getQuery(TEST), + sourceQuery.getQueryId(CONTROL), + sourceQuery.getQueryId(TEST), sourceQuery.getControlConfiguration().applyOverrides(controlOverrides), sourceQuery.getTestConfiguration().applyOverrides(testOverrides))) .collect(toImmutableList()); @@ -354,26 +357,25 @@ private void reportProgressUntilFinished() VerificationResult result = completionService.take().get(); Optional event = result.getEvent(); completed++; - if (!event.isPresent()) { - statusCount.compute(SKIPPED, (status, count) -> count == null ? 1 : count + 1); - } - else { + if (event.isPresent()) { statusCount.compute(EventStatus.valueOf(event.get().getStatus()), (status, count) -> count == null ? 1 : count + 1); postEvent(event.get()); } if (result.shouldResubmit()) { + statusCount.compute(RESUBMITTED, (status, count) -> count == null ? 1 : count + 1); resubmit(result.getVerification()); } double progress = ((double) completed) / queriesSubmitted.get() * 100; - if (progress - lastProgress > 0.5 || completed == queriesSubmitted.get()) { + if (progress - lastProgress > 0.2 || completed == queriesSubmitted.get()) { log.info( - "Progress: %s succeeded, %s skipped, %s resolved, %s failed, %.2f%% done", + "Progress: %s succeeded, %s skipped, %s resolved, %s failed, %s resubmitted, %.2f%% done", statusCount.getOrDefault(SUCCEEDED, 0), statusCount.getOrDefault(SKIPPED, 0), statusCount.getOrDefault(FAILED_RESOLVED, 0), statusCount.getOrDefault(FAILED, 0), + statusCount.getOrDefault(RESUBMITTED, 0), progress); lastProgress = progress; } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java index b49a411a0f032..5bcf2b234de41 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java @@ -47,7 +47,9 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_OFFLINE; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_READ_ONLY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_READ_ONLY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR; @@ -70,7 +72,6 @@ import static com.facebook.presto.spi.StandardErrorCode.TOO_MANY_REQUESTS_FAILED; import static com.facebook.presto.verifier.framework.QueryStage.CONTROL_SETUP; import static com.facebook.presto.verifier.framework.QueryStage.DESCRIBE; -import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; import static com.facebook.presto.verifier.framework.QueryStage.TEST_SETUP; import static com.google.common.base.Functions.identity; import static com.google.common.base.MoreObjects.toStringHelper; @@ -148,10 +149,12 @@ public static Builder defaultBuilder() .addResubmittedError(HIVE_TABLE_DROPPED_DURING_QUERY) .addResubmittedError(CLUSTER_OUT_OF_MEMORY) .addResubmittedError(ADMINISTRATIVELY_PREEMPTED) + .addResubmittedError(HIVE_TABLE_READ_ONLY) + .addResubmittedError(HIVE_PARTITION_READ_ONLY) + .addResubmittedError(HIVE_PARTITION_OFFLINE) // Conditional Resubmitted Errors .addResubmittedError(SYNTAX_ERROR, Optional.of(CONTROL_SETUP), Optional.of(TABLE_ALREADY_EXISTS_PATTERN)) - .addResubmittedError(SYNTAX_ERROR, Optional.of(TEST_SETUP), Optional.of(TABLE_ALREADY_EXISTS_PATTERN)) - .addResubmittedError(HIVE_PARTITION_OFFLINE, Optional.of(TEST_MAIN), Optional.empty()); + .addResubmittedError(SYNTAX_ERROR, Optional.of(TEST_SETUP), Optional.of(TABLE_ALREADY_EXISTS_PATTERN)); } public QueryException createException(QueryStage queryStage, QueryActionStats queryActionStats, SQLException cause) diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionStats.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionStats.java index 73fc6aac72f20..944f3bcb117ca 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionStats.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionStats.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.prestoaction; +import com.facebook.presto.execution.QueryState; import com.facebook.presto.jdbc.QueryStats; import java.util.Optional; @@ -26,6 +27,12 @@ public class QueryActionStats private final Optional queryStats; private final Optional extraStats; + public static QueryActionStats queryIdStats(String queryId) + { + return new QueryActionStats(Optional.of(new QueryStats(queryId, QueryState.FINISHED.toString(), false, false, false, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Optional.empty())), Optional.empty()); + } + public QueryActionStats(Optional queryStats, Optional extraStats) { this.queryStats = requireNonNull(queryStats, "queryStats is null"); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriteConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriteConfig.java index 9c524f959943a..c74b5a66068c9 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriteConfig.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriteConfig.java @@ -31,6 +31,21 @@ public class QueryRewriteConfig private QualifiedName tablePrefix = QualifiedName.of("tmp_verifier"); private Map tableProperties = ImmutableMap.of(); + private boolean reuseTable; + + public boolean isReuseTable() + { + return reuseTable; + } + + @ConfigDescription("If true, reuse the output table of the source query. Otherwise, run the query and write to a temporary shadow table.") + @Config("reuse-table") + public QueryRewriteConfig setReuseTable(boolean reuseTable) + { + this.reuseTable = reuseTable; + return this; + } + @NotNull public QualifiedName getTablePrefix() { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java index c5949e369079a..dbd7048535837 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java @@ -43,6 +43,7 @@ import com.facebook.presto.sql.tree.SingleColumn; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.verifier.framework.ClusterType; +import com.facebook.presto.verifier.framework.QueryConfiguration; import com.facebook.presto.verifier.framework.QueryException; import com.facebook.presto.verifier.framework.QueryObjectBundle; import com.facebook.presto.verifier.prestoaction.PrestoAction; @@ -98,6 +99,7 @@ public class QueryRewriter private final PrestoAction prestoAction; private final Map prefixes; private final Map> tableProperties; + private final Map reuseTables; private final Optional functionCallRewriter; public QueryRewriter( @@ -105,9 +107,10 @@ public QueryRewriter( TypeManager typeManager, PrestoAction prestoAction, Map tablePrefixes, - Map> tableProperties) + Map> tableProperties, + Map reuseTables) { - this(sqlParser, typeManager, prestoAction, tablePrefixes, tableProperties, ImmutableMultimap.of()); + this(sqlParser, typeManager, prestoAction, tablePrefixes, tableProperties, reuseTables, ImmutableMultimap.of()); } public QueryRewriter( @@ -116,6 +119,7 @@ public QueryRewriter( PrestoAction prestoAction, Map tablePrefixes, Map> tableProperties, + Map reuseTables, Multimap functionSubstitutes) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); @@ -123,26 +127,46 @@ public QueryRewriter( this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); this.prefixes = ImmutableMap.copyOf(tablePrefixes); this.tableProperties = ImmutableMap.copyOf(tableProperties); + this.reuseTables = ImmutableMap.copyOf(reuseTables); this.functionCallRewriter = FunctionCallRewriter.getInstance(functionSubstitutes, typeManager); } - public QueryObjectBundle rewriteQuery(@Language("SQL") String query, ClusterType clusterType) + public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfiguration queryConfiguration, ClusterType clusterType) + { + return rewriteQuery(query, queryConfiguration, clusterType, false); + } + + public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfiguration queryConfiguration, ClusterType clusterType, boolean reuseTable) { checkState(prefixes.containsKey(clusterType), "Unsupported cluster type: %s", clusterType); Statement statement = sqlParser.createStatement(query, PARSING_OPTIONS); QualifiedName prefix = prefixes.get(clusterType); List properties = tableProperties.get(clusterType); + boolean shouldReuseTable = reuseTable && reuseTables.get(clusterType) && queryConfiguration.isReusableTable(); + if (statement instanceof CreateTableAsSelect) { CreateTableAsSelect createTableAsSelect = (CreateTableAsSelect) statement; - QualifiedName temporaryTableName = generateTemporaryName(Optional.of(createTableAsSelect.getName()), prefix); Query createQuery = createTableAsSelect.getQuery(); + Optional functionSubstitutions = Optional.empty(); if (functionCallRewriter.isPresent()) { FunctionCallRewriter.RewriterResult rewriterResult = functionCallRewriter.get().rewrite(createQuery); createQuery = (Query) rewriterResult.getRewrittenNode(); functionSubstitutions = rewriterResult.getSubstitutions(); } + if (shouldReuseTable && !functionSubstitutions.isPresent()) { + return new QueryObjectBundle( + createTableAsSelect.getName(), + ImmutableList.of(), + createTableAsSelect, + ImmutableList.of(), + clusterType, + Optional.empty(), + true); + } + + QualifiedName temporaryTableName = generateTemporaryName(Optional.of(createTableAsSelect.getName()), prefix); return new QueryObjectBundle( temporaryTableName, ImmutableList.of(), @@ -156,19 +180,32 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, ClusterType createTableAsSelect.getComment()), ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, - functionSubstitutions); + functionSubstitutions, + false); } if (statement instanceof Insert) { Insert insert = (Insert) statement; QualifiedName originalTableName = insert.getTarget(); - QualifiedName temporaryTableName = generateTemporaryName(Optional.of(originalTableName), prefix); Query insertQuery = insert.getQuery(); + Optional functionSubstitutions = Optional.empty(); if (functionCallRewriter.isPresent()) { FunctionCallRewriter.RewriterResult rewriterResult = functionCallRewriter.get().rewrite(insertQuery); insertQuery = (Query) rewriterResult.getRewrittenNode(); functionSubstitutions = rewriterResult.getSubstitutions(); } + if (shouldReuseTable && !functionSubstitutions.isPresent()) { + return new QueryObjectBundle( + originalTableName, + ImmutableList.of(), + insert, + ImmutableList.of(), + clusterType, + Optional.empty(), + true); + } + + QualifiedName temporaryTableName = generateTemporaryName(Optional.of(originalTableName), prefix); return new QueryObjectBundle( temporaryTableName, ImmutableList.of( @@ -184,21 +221,23 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, ClusterType insertQuery), ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, - functionSubstitutions); + functionSubstitutions, + false); } if (statement instanceof Query) { - QualifiedName temporaryTableName = generateTemporaryName(Optional.empty(), prefix); Query queryBody = (Query) statement; + Optional functionSubstitutions = Optional.empty(); if (functionCallRewriter.isPresent()) { FunctionCallRewriter.RewriterResult rewriterResult = functionCallRewriter.get().rewrite(queryBody); queryBody = (Query) rewriterResult.getRewrittenNode(); functionSubstitutions = rewriterResult.getSubstitutions(); } + + QualifiedName temporaryTableName = generateTemporaryName(Optional.empty(), prefix); ResultSetMetaData metadata = getResultMetadata(queryBody); List columnAliases = generateStorageColumnAliases(metadata); queryBody = rewriteNonStorableColumns(queryBody, metadata); - return new QueryObjectBundle( temporaryTableName, ImmutableList.of(), @@ -212,7 +251,8 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, ClusterType Optional.empty()), ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, - functionSubstitutions); + functionSubstitutions, + false); } if (statement instanceof CreateView) { CreateView createView = (CreateView) statement; @@ -247,7 +287,8 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, ClusterType createView.getSecurity()), ImmutableList.of(new DropView(temporaryViewName, true)), clusterType, - Optional.empty()); + Optional.empty(), + false); } if (statement instanceof CreateTable) { CreateTable createTable = (CreateTable) statement; @@ -263,7 +304,8 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, ClusterType createTable.getComment()), ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, - Optional.empty()); + Optional.empty(), + false); } throw new IllegalStateException(format("Unsupported query type: %s", statement.getClass())); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java index 13dbba94d692c..8b8d0ff8f40e2 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java @@ -52,6 +52,8 @@ public class VerificationQueryRewriterFactory private final QualifiedName testTablePrefix; private final List controlTableProperties; private final List testTableProperties; + private final boolean controlReuseTable; + private final boolean testReuseTable; private final Multimap functionSubstitutes; @@ -69,6 +71,8 @@ public VerificationQueryRewriterFactory( this.testTablePrefix = requireNonNull(testConfig.getTablePrefix(), "testTablePrefix is null"); this.controlTableProperties = constructProperties(controlConfig.getTableProperties()); this.testTableProperties = constructProperties(testConfig.getTableProperties()); + this.controlReuseTable = controlConfig.isReuseTable(); + this.testReuseTable = testConfig.isReuseTable(); this.functionSubstitutes = verifierConfig.getFunctionSubstitutes(); } @@ -81,6 +85,7 @@ public QueryRewriter create(PrestoAction prestoAction) prestoAction, ImmutableMap.of(CONTROL, controlTablePrefix, TEST, testTablePrefix), ImmutableMap.of(CONTROL, controlTableProperties, TEST, testTableProperties), + ImmutableMap.of(CONTROL, controlReuseTable, TEST, testReuseTable), functionSubstitutes); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java index ba94907900114..1791c29f4ed5f 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java @@ -41,20 +41,26 @@ public class PrestoQuerySourceQuerySupplier resultSet.getString("name"), resultSet.getString("control_query"), resultSet.getString("test_query"), + Optional.ofNullable(resultSet.getString("control_query_id")), + Optional.ofNullable(resultSet.getString("test_query_id")), new QueryConfiguration( resultSet.getString("control_catalog"), resultSet.getString("control_schema"), Optional.ofNullable(resultSet.getString("control_username")), Optional.ofNullable(resultSet.getString("control_password")), Optional.ofNullable(resultSet.getString("control_session_properties")) - .map(StringToStringMapColumnMapper.CODEC::fromJson)), + .map(StringToStringMapColumnMapper.CODEC::fromJson), + Optional.ofNullable(resultSet.getString("control_client_tags")) + .map(StringListColumnMapper.CODEC::fromJson)), new QueryConfiguration( resultSet.getString("test_catalog"), resultSet.getString("test_schema"), Optional.ofNullable(resultSet.getString("test_username")), Optional.ofNullable(resultSet.getString("test_password")), Optional.ofNullable(resultSet.getString("test_session_properties")) - .map(StringToStringMapColumnMapper.CODEC::fromJson)))); + .map(StringToStringMapColumnMapper.CODEC::fromJson), + Optional.ofNullable(resultSet.getString("test_client_tags")) + .map(StringListColumnMapper.CODEC::fromJson)))); private final PrestoAction helperAction; private final SqlParser sqlParser; @@ -67,7 +73,7 @@ public PrestoQuerySourceQuerySupplier( PrestoQuerySourceQueryConfig config) { this.helperAction = helperActionFactory.create( - new QueryConfiguration(config.getCatalog(), config.getSchema(), config.getUsername(), config.getPassword(), Optional.empty()), + new QueryConfiguration(config.getCatalog(), config.getSchema(), config.getUsername(), config.getPassword(), Optional.empty(), Optional.empty()), VerificationContext.create("", "")); this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.query = requireNonNull(config.getQuery(), "query is null"); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/StringListColumnMapper.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/StringListColumnMapper.java new file mode 100644 index 0000000000000..0e9d9f99e10e2 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/StringListColumnMapper.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.source; + +import com.facebook.airlift.json.JsonCodec; +import org.jdbi.v3.core.mapper.ColumnMapper; +import org.jdbi.v3.core.statement.StatementContext; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; + +public class StringListColumnMapper + implements ColumnMapper> +{ + public static final JsonCodec> CODEC = listJsonCodec(String.class); + + @Override + public List map(ResultSet resultSet, int columnNumber, StatementContext ctx) + throws SQLException + { + String columnValue = resultSet.getString(columnNumber); + return columnValue == null ? null : CODEC.fromJson(columnValue); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java index 72407527ea859..ef088b2ce5e56 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java @@ -25,6 +25,7 @@ import java.util.List; @RegisterColumnMapper(StringToStringMapColumnMapper.class) +@RegisterColumnMapper(StringListColumnMapper.class) public interface VerifierDao { @SqlUpdate("CREATE TABLE (\n" + @@ -34,32 +35,40 @@ public interface VerifierDao " control_catalog varchar(256) NOT NULL,\n" + " control_schema varchar(256) NOT NULL,\n" + " control_query text NOT NULL,\n" + + " control_query_id mediumtext DEFAULT NULL,\n" + " control_username varchar(256) DEFAULT NULL,\n" + " control_password varchar(256) DEFAULT NULL,\n" + " control_session_properties text DEFAULT NULL,\n" + + " control_client_tags text DEFAULT NULL,\n" + " test_catalog varchar(256) NOT NULL,\n" + " test_schema varchar(256) NOT NULL,\n" + " test_query text NOT NULL,\n" + + " test_query_id mediumtext DEFAULT NULL,\n" + " test_username varchar(256) DEFAULT NULL,\n" + " test_password varchar(256) DEFAULT NULL,\n" + - " test_session_properties text DEFAULT NULL)") + " test_session_properties text DEFAULT NULL,\n" + + " test_client_tags text DEFAULT NULL)") void createVerifierQueriesTable(@Define("table_name") String tableName); @SqlQuery("SELECT\n" + " suite,\n" + " name,\n" + " control_query,\n" + + " name control_query_id,\n" + " control_catalog,\n" + " control_schema,\n" + " control_username,\n" + " control_password,\n" + " control_session_properties,\n" + + " control_client_tags,\n" + " test_query,\n" + + " test_query_id,\n" + " test_catalog,\n" + " test_schema,\n" + " test_username,\n" + " test_password,\n" + - " test_session_properties\n" + + " test_session_properties,\n" + + " test_client_tags\n" + "FROM\n" + " \n" + "WHERE\n" + diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java index f48282842bb92..4da570f0e7fbc 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java @@ -71,7 +71,8 @@ public class VerifierTestUtil ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build()), ImmutableList.of(), CONTROL, - Optional.empty()); + Optional.empty(), + false); private static final MySqlOptions MY_SQL_OPTIONS = MySqlOptions.builder() .setCommandTimeout(new Duration(90, SECONDS)) diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java index 33b1dfb076421..d6b90e425ef53 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java @@ -62,7 +62,7 @@ public abstract class AbstractVerificationTest protected static final String SUITE = "test-suite"; protected static final String NAME = "test-query"; protected static final String TEST_ID = "test-id"; - protected static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty()); + protected static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), true); protected static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build(); protected static final String CONTROL_TABLE_PREFIX = "tmp_verifier_c"; protected static final String TEST_TABLE_PREFIX = "tmp_verifier_t"; @@ -72,6 +72,8 @@ public abstract class AbstractVerificationTest protected static VerificationSettings saveSnapshotSettings; protected static VerificationSettings queryBankModeSettings; + protected static VerificationSettings reuseTableSettings; + private final StandaloneQueryRunner queryRunner; private final Injector injector; @@ -98,6 +100,8 @@ public AbstractVerificationTest() saveSnapshotSettings.saveSnapshot = Optional.of(true); queryBankModeSettings = new VerificationSettings(); queryBankModeSettings.runningMode = Optional.of("query-bank"); + reuseTableSettings = new VerificationSettings(); + reuseTableSettings.reuseTable = Optional.of(true); } @AfterClass @@ -122,7 +126,12 @@ public SqlParser getSqlParser() protected SourceQuery getSourceQuery(String controlQuery, String testQuery) { - return new SourceQuery(SUITE, NAME, controlQuery, testQuery, QUERY_CONFIGURATION, QUERY_CONFIGURATION); + return new SourceQuery(SUITE, NAME, controlQuery, testQuery, Optional.empty(), Optional.empty(), QUERY_CONFIGURATION, QUERY_CONFIGURATION); + } + + protected SourceQuery getSourceQuery(String controlQuery, String testQuery, String controlQueryId, String testQueryId) + { + return new SourceQuery(SUITE, NAME, controlQuery, testQuery, Optional.of(controlQueryId), Optional.of(testQueryId), QUERY_CONFIGURATION, QUERY_CONFIGURATION); } protected Optional runExplain(String controlQuery, String testQuery) @@ -145,6 +154,11 @@ protected Optional runVerification(String controlQuery, Stri return verify(getSourceQuery(controlQuery, testQuery), false, Optional.empty(), Optional.of(settings)); } + protected Optional runVerification(String controlQuery, String testQuery, String controlQueryId, String testQueryId, VerificationSettings settings) + { + return verify(getSourceQuery(controlQuery, testQuery, controlQueryId, testQueryId), false, Optional.empty(), Optional.of(settings)); + } + protected Optional verify(SourceQuery sourceQuery, boolean explain) { return verify(sourceQuery, explain, Optional.empty(), Optional.empty()); @@ -191,13 +205,19 @@ private Optional verify( settings.saveSnapshot.ifPresent(verifierConfig::setSaveSnapshot); settings.functionSubstitutes.ifPresent(verifierConfig::setFunctionSubstitutes); }); + QueryRewriteConfig controlRewriteConfig = new QueryRewriteConfig().setTablePrefix(CONTROL_TABLE_PREFIX); + QueryRewriteConfig testRewriteConfig = new QueryRewriteConfig().setTablePrefix(TEST_TABLE_PREFIX); + verificationSettings.ifPresent(settings -> { + settings.reuseTable.ifPresent(controlRewriteConfig::setReuseTable); + settings.reuseTable.ifPresent(testRewriteConfig::setReuseTable); + }); TypeManager typeManager = createTypeManager(); PrestoAction prestoAction = mockPrestoAction.orElseGet(() -> getPrestoAction(Optional.of(sourceQuery.getControlConfiguration()))); QueryRewriterFactory queryRewriterFactory = new VerificationQueryRewriterFactory( sqlParser, typeManager, - new QueryRewriteConfig().setTablePrefix(CONTROL_TABLE_PREFIX), - new QueryRewriteConfig().setTablePrefix(TEST_TABLE_PREFIX), + controlRewriteConfig, + testRewriteConfig, verifierConfig); VerificationFactory verificationFactory = new VerificationFactory( @@ -225,14 +245,15 @@ public VerificationSettings() runningMode = Optional.empty(); saveSnapshot = Optional.empty(); functionSubstitutes = Optional.empty(); + reuseTable = Optional.empty(); } Optional concurrentControlAndTest; Optional skipControl; Optional runningMode; Optional saveSnapshot; - Optional functionSubstitutes; + Optional reuseTable; } public static class MockSnapshotSupplierAndConsumer diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java index 271723c8f7511..90858af58f696 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java @@ -213,6 +213,25 @@ public void testControlFailed() "CONTROL SETUP query failed on CONTROL cluster:\n.*")); } + @Test + public void testReuseTable() + { + getQueryRunner().execute("CREATE TABLE test_reuse_table (test_column INT)"); + String testQuery = "INSERT INTO test_reuse_table SELECT 1"; + getQueryRunner().execute(testQuery); + String testQueryId = "test_query_id"; + + getQueryRunner().execute("CREATE TABLE control_reuse_table (test_column INT)"); + String controlQuery = "INSERT INTO control_reuse_table SELECT 1"; + getQueryRunner().execute(controlQuery); + String controlQueryId = "control_query_id"; + + Optional event = runVerification(testQuery, controlQuery, controlQueryId, testQueryId, reuseTableSettings); + assertTrue(event.get().getControlQueryInfo().getIsReuseTable()); + assertTrue(event.get().getTestQueryInfo().getIsReuseTable()); + assertEvent(event.get(), SUCCEEDED, Optional.empty(), Optional.empty(), Optional.empty()); + } + @Test public void testNonDeterministic() { @@ -357,8 +376,9 @@ public void testChecksumQueryCompilerError() @Test public void testExecutionTimeSessionProperty() { - QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of(QUERY_MAX_EXECUTION_TIME, "20m"))); - SourceQuery sourceQuery = new SourceQuery(SUITE, NAME, "SELECT 1.0", "SELECT 1.00001", configuration, configuration); + QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of(QUERY_MAX_EXECUTION_TIME, + "20m")), Optional.empty()); + SourceQuery sourceQuery = new SourceQuery(SUITE, NAME, "SELECT 1.0", "SELECT 1.00001", Optional.empty(), Optional.empty(), configuration, configuration); Optional event = verify(sourceQuery, false); assertTrue(event.isPresent()); assertEvent(event.get(), SUCCEEDED, Optional.empty(), Optional.empty(), Optional.empty()); @@ -437,16 +457,28 @@ private static void assertSuccessQueryInfo(QueryType queryType, QueryInfo queryI assertNotNull(queryInfo.getSessionProperties()); assertNotNull(queryInfo.getSetupQueries()); assertNotNull(queryInfo.getTeardownQueries()); - assertEquals(queryInfo.getTeardownQueries().size(), 1); assertNotNull(queryInfo.getQueryId()); assertNotNull(queryInfo.getSetupQueryIds()); assertNotNull(queryInfo.getTeardownQueryIds()); - assertEquals(queryInfo.getTeardownQueryIds().size(), 1); + if (queryInfo.getIsReuseTable()) { + assertEquals(queryInfo.getTeardownQueries().size(), 0); + assertEquals(queryInfo.getTeardownQueryIds().size(), 0); + } + else { + assertEquals(queryInfo.getTeardownQueries().size(), 1); + assertEquals(queryInfo.getTeardownQueryIds().size(), 1); + } if (queryType == QueryType.INSERT) { - assertEquals(queryInfo.getSetupQueries().size(), 1); - assertEquals(queryInfo.getSetupQueryIds().size(), 1); + if (queryInfo.getIsReuseTable()) { + assertEquals(queryInfo.getSetupQueries().size(), 0); + assertEquals(queryInfo.getSetupQueryIds().size(), 0); + } + else { + assertEquals(queryInfo.getSetupQueries().size(), 1); + assertEquals(queryInfo.getSetupQueryIds().size(), 1); + } } assertNotNull(queryInfo.getOutputTableName()); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java index 16d5ae36e1505..3fd179d4c7835 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java @@ -66,7 +66,7 @@ private static boolean isMutableCatalogReferenced(DeterminismAnalyzer determinis private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatalogPattern) { - QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty()); + QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); VerificationContext verificationContext = VerificationContext.create(SUITE, NAME); VerifierConfig verifierConfig = new VerifierConfig().setTestId("test-id"); RetryConfig retryConfig = new RetryConfig(); @@ -88,9 +88,10 @@ private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatal typeManager, prestoAction, ImmutableMap.of(CONTROL, QualifiedName.of("tmp_verifier_c"), TEST, QualifiedName.of("tmp_verifier_t")), - ImmutableMap.of()); + ImmutableMap.of(), + ImmutableMap.of(CONTROL, false, TEST, false)); ChecksumValidator checksumValidator = createChecksumValidator(verifierConfig); - SourceQuery sourceQuery = new SourceQuery("test", "", "", "", configuration, configuration); + SourceQuery sourceQuery = new SourceQuery("test", "", "", "", Optional.empty(), Optional.empty(), configuration, configuration); return new DeterminismAnalyzer( sourceQuery, diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java index 2c71d52bbc23d..f03b305fde5fe 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java @@ -13,15 +13,19 @@ */ package com.facebook.presto.verifier.framework; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.util.List; import java.util.Map; import java.util.Optional; import static com.facebook.airlift.json.JsonCodec.mapJsonCodec; import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.facebook.presto.testing.assertions.Assert.assertFalse; +import static com.facebook.presto.testing.assertions.Assert.assertTrue; import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.NO_ACTION; import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.OVERRIDE; import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.SUBSTITUTE; @@ -43,14 +47,18 @@ public class TestQueryConfiguration private static final Map SESSION_PROPERTIES_OVERRIDE = ImmutableMap.of("property_1", "value_x", "property_3", "value_3"); private static final String SESSION_PROPERTIES_OVERRIDE_CONFIG = mapJsonCodec(String.class, String.class).toJson(SESSION_PROPERTIES_OVERRIDE); - private static final QueryConfiguration CONFIGURATION_1 = new QueryConfiguration(CATALOG, SCHEMA, Optional.of(USERNAME), Optional.of(PASSWORD), Optional.of(SESSION_PROPERTIES)); - private static final QueryConfiguration CONFIGURATION_2 = new QueryConfiguration(CATALOG, SCHEMA, Optional.empty(), Optional.empty(), Optional.empty()); + private static final List CLIENT_TAGS = ImmutableList.of(QueryConfiguration.CLIENT_TAG_OUTPUT_RETAINED); + + private static final QueryConfiguration CONFIGURATION_1 = new QueryConfiguration(CATALOG, SCHEMA, Optional.of(USERNAME), Optional.of(PASSWORD), + Optional.of(SESSION_PROPERTIES), Optional.of(CLIENT_TAGS)); + private static final QueryConfiguration CONFIGURATION_2 = new QueryConfiguration(CATALOG, SCHEMA, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); private static final QueryConfiguration CONFIGURATION_FULL_OVERRIDE = new QueryConfiguration( CATALOG_OVERRIDE, SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), - Optional.of(SESSION_PROPERTIES_OVERRIDE)); + Optional.of(SESSION_PROPERTIES_OVERRIDE), + Optional.of(CLIENT_TAGS)); private QueryConfigurationOverridesConfig overrides; @@ -82,13 +90,15 @@ public void testOverrides() SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), - Optional.of(SESSION_PROPERTIES))); + Optional.of(SESSION_PROPERTIES), + Optional.of(CLIENT_TAGS))); assertEquals(CONFIGURATION_2.applyOverrides(overrides), new QueryConfiguration( CATALOG_OVERRIDE, SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), + Optional.empty(), Optional.empty())); } @@ -97,22 +107,38 @@ public void testSessionPropertyOverride() { overrides.setSessionPropertiesOverrideStrategy(OVERRIDE); assertEquals(CONFIGURATION_1.applyOverrides(overrides), CONFIGURATION_FULL_OVERRIDE); - assertEquals(CONFIGURATION_2.applyOverrides(overrides), CONFIGURATION_FULL_OVERRIDE); + QueryConfiguration overridden = new QueryConfiguration( + CATALOG_OVERRIDE, + SCHEMA_OVERRIDE, + Optional.of(USERNAME_OVERRIDE), + Optional.of(PASSWORD_OVERRIDE), + Optional.of(SESSION_PROPERTIES_OVERRIDE), + Optional.empty()); + assertEquals(CONFIGURATION_2.applyOverrides(overrides), overridden); } @Test public void testSessionPropertySubstitute() { overrides.setSessionPropertiesOverrideStrategy(SUBSTITUTE); - QueryConfiguration substituted = new QueryConfiguration( + QueryConfiguration substituted1 = new QueryConfiguration( CATALOG_OVERRIDE, SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), - Optional.of(ImmutableMap.of("property_1", "value_x", "property_2", "value_2", "property_3", "value_3"))); + Optional.of(ImmutableMap.of("property_1", "value_x", "property_2", "value_2", "property_3", "value_3")), + Optional.of(CLIENT_TAGS)); + + assertEquals(CONFIGURATION_1.applyOverrides(overrides), substituted1); - assertEquals(CONFIGURATION_1.applyOverrides(overrides), substituted); - assertEquals(CONFIGURATION_2.applyOverrides(overrides), CONFIGURATION_FULL_OVERRIDE); + QueryConfiguration substituted2 = new QueryConfiguration( + CATALOG_OVERRIDE, + SCHEMA_OVERRIDE, + Optional.of(USERNAME_OVERRIDE), + Optional.of(PASSWORD_OVERRIDE), + Optional.of(SESSION_PROPERTIES_OVERRIDE), + Optional.empty()); + assertEquals(CONFIGURATION_2.applyOverrides(overrides), substituted2); } @Test @@ -126,7 +152,8 @@ public void testSessionPropertyRemovalWithOverrides() SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), - Optional.of(ImmutableMap.of("property_3", "value_3"))); + Optional.of(ImmutableMap.of("property_3", "value_3")), + Optional.of(CLIENT_TAGS)); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } @@ -141,7 +168,8 @@ public void testSessionPropertySubstituteAndRemove() SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), - Optional.of(SESSION_PROPERTIES_OVERRIDE)); + Optional.of(SESSION_PROPERTIES_OVERRIDE), + Optional.of(CLIENT_TAGS)); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } @@ -156,8 +184,16 @@ public void testSessionPropertyRemoval() SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), - Optional.of(ImmutableMap.of("property_1", "value_1"))); + Optional.of(ImmutableMap.of("property_1", "value_1")), + Optional.of(CLIENT_TAGS)); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } + + @Test + public void testIsReusableTable() + { + assertTrue(CONFIGURATION_1.isReusableTable()); + assertFalse(CONFIGURATION_2.isReusableTable()); + } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java index 70da6ae069151..4361fe7518790 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java @@ -115,12 +115,14 @@ public List getEvents() private static final String NAME = "test-query"; private static final QualifiedName TABLE_PREFIX = QualifiedName.of("tmp_verifier"); private static final SqlParser SQL_PARSER = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(AT_SIGN, COLON)); - private static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration("test", "di", Optional.of("user"), Optional.empty(), Optional.empty()); + private static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration("test", "di", Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); private static final SourceQuery SOURCE_QUERY = new SourceQuery( SUITE, NAME, "SELECT 1", "SELECT 2", + Optional.of("control_query_id"), + Optional.of("test_query_id"), QUERY_CONFIGURATION, QUERY_CONFIGURATION); private static final VerifierConfig VERIFIER_CONFIG = new VerifierConfig().setTestId("test"); @@ -197,7 +199,7 @@ public void testVerifierError() private static SourceQuery createSourceQuery(String name, String controlQuery, String testQuery) { - return new SourceQuery(SUITE, name, controlQuery, testQuery, QUERY_CONFIGURATION, QUERY_CONFIGURATION); + return new SourceQuery(SUITE, name, controlQuery, testQuery, Optional.empty(), Optional.empty(), QUERY_CONFIGURATION, QUERY_CONFIGURATION); } private static void assertSkippedEvent(VerifierQueryEvent event, String name, SkippedReason skippedReason) @@ -216,7 +218,9 @@ private VerificationManager getVerificationManager(List sourceQueri new VerificationFactory( SQL_PARSER, (sourceQuery, verificationContext) -> new QueryActions(prestoAction, prestoAction, prestoAction), - presto -> new QueryRewriter(SQL_PARSER, createTypeManager(), presto, ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX), ImmutableMap.of()), + presto -> new QueryRewriter(SQL_PARSER, createTypeManager(), presto, ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX), + ImmutableMap.of(), + ImmutableMap.of(CONTROL, false, TEST, false)), new FailureResolverManagerFactory(ImmutableSet.of(), ImmutableSet.of()), createChecksumValidator(verifierConfig), PrestoExceptionClassifier.defaultBuilder().build(), diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java index 6a5b2f6685fa1..7869b152a91de 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java @@ -56,7 +56,7 @@ public class TestJdbcPrestoAction private static final String SUITE = "test-suite"; private static final String NAME = "test-query"; private static final QueryStage QUERY_STAGE = CONTROL_MAIN; - private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty()); + private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DECIMAL).build(); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java index 8caf0187a5765..e6b749a1dc236 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java @@ -68,6 +68,7 @@ private static QueryObjectBundle createBundle(String query) sqlParser.createStatement(query, PARSING_OPTIONS), ImmutableList.of(), CONTROL, - Optional.empty()); + Optional.empty(), + false); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java index 66c2514921193..827c8a8929709 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java @@ -88,7 +88,8 @@ public QueryResult execute(Statement statement, QueryStage queryStage, Re ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build()), ImmutableList.of(), TEST, - Optional.empty()); + Optional.empty(), + false); private static final QueryException HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION = new PrestoQueryException( new RuntimeException(), false, diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriteConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriteConfig.java index 119ee21805eea..89f7266cbad16 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriteConfig.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriteConfig.java @@ -30,7 +30,8 @@ public void testDefault() { assertRecordedDefaults(recordDefaults(QueryRewriteConfig.class) .setTablePrefix("tmp_verifier") - .setTableProperties(null)); + .setTableProperties(null) + .setReuseTable(false)); } @Test @@ -39,10 +40,12 @@ public void testExplicitPropertyMappings() Map properties = new ImmutableMap.Builder() .put("table-prefix", "local.tmp") .put("table-properties", "{\"retention\":21}") + .put("reuse-table", "true") .build(); QueryRewriteConfig expected = new QueryRewriteConfig() .setTablePrefix("local.tmp") - .setTableProperties("{\"retention\":21}"); + .setTableProperties("{\"retention\":21}") + .setReuseTable(true); assertFullMapping(properties, expected); assertEquals(expected.getTableProperties(), ImmutableMap.of("retention", 21)); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java index 0cf6cdbb91ba9..95b060e73b81d 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java @@ -35,6 +35,7 @@ import com.facebook.presto.verifier.prestoaction.QueryActionsConfig; import com.facebook.presto.verifier.retry.RetryConfig; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import org.intellij.lang.annotations.Language; import org.testng.annotations.AfterClass; @@ -42,6 +43,7 @@ import org.testng.annotations.Test; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.facebook.presto.sql.SqlFormatter.formatSql; @@ -53,6 +55,7 @@ import static com.facebook.presto.verifier.VerifierTestUtil.createTypeManager; import static com.facebook.presto.verifier.VerifierTestUtil.setupPresto; import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; import static com.facebook.presto.verifier.rewrite.FunctionCallRewriter.FunctionCallSubstitute; import static com.facebook.presto.verifier.rewrite.FunctionCallRewriter.validateAndConstructFunctionCallSubstituteMap; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -67,7 +70,13 @@ public class TestQueryRewriter { private static final String SUITE = "test-suite"; private static final String NAME = "test-query"; - private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty()); + private static final QueryConfiguration CONFIGURATION = new QueryConfiguration( + CATALOG, + SCHEMA, + Optional.of("user"), + Optional.empty(), + Optional.empty(), + Optional.empty()); private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build(); private static final QueryRewriteConfig QUERY_REWRITE_CONFIG = new QueryRewriteConfig() .setTablePrefix("local.tmp") @@ -229,7 +238,7 @@ public void testTemporaryTableName() @Test public void testRewriteDate() { - QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT date '2020-01-01', date(now()) today", CONTROL); + QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT date '2020-01-01', date(now()) today", CONFIGURATION, CONTROL); assertCreateTableAs(queryBundle.getQuery(), "SELECT\n" + " CAST(date '2020-01-01' AS timestamp)\n" + ", CAST(date(now()) AS timestamp) today"); @@ -238,7 +247,7 @@ public void testRewriteDate() @Test public void testRewriteTime() { - QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT time '12:34:56', time '12:34:56' now", CONTROL); + QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT time '12:34:56', time '12:34:56' now", CONFIGURATION, CONTROL); assertCreateTableAs(queryBundle.getQuery(), "SELECT\n" + " CAST(time '12:34:56' AS timestamp)\n" + ", CAST(time '12:34:56' AS timestamp) now"); @@ -247,7 +256,7 @@ public void testRewriteTime() @Test public void testRewriteTimestampWithTimeZone() { - QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT now(), now() now", CONTROL); + QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT now(), now() now", CONFIGURATION, CONTROL); assertCreateTableAs(queryBundle.getQuery(), "SELECT\n" + " CAST(now() AS varchar)\n" + ", CAST(now() AS varchar) now"); @@ -256,7 +265,7 @@ public void testRewriteTimestampWithTimeZone() @Test public void testRewriteUnknown() { - QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT null, null unknown", CONTROL); + QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT null, null unknown", CONFIGURATION, CONTROL); assertCreateTableAs(queryBundle.getQuery(), "SELECT\n" + " CAST(null AS bigint)\n" + ", CAST(null AS bigint) unknown"); @@ -265,7 +274,7 @@ public void testRewriteUnknown() @Test public void testRewriteDecimal() { - QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT decimal '1.2', decimal '1.2' d", CONTROL); + QueryBundle queryBundle = getQueryRewriter().rewriteQuery("SELECT decimal '1.2', decimal '1.2' d", CONFIGURATION, CONTROL); assertCreateTableAs(queryBundle.getQuery(), "SELECT\n" + " CAST(decimal '1.2' AS double)\n" + ", CAST(decimal '1.2' AS double) d"); @@ -284,7 +293,7 @@ public void testRewriteNonStorableStructuredTypes() " ]\n" + " ),\n" + " ROW(NULL)", - CONTROL); + CONFIGURATION, CONTROL); assertCreateTableAs( queryBundle.getQuery(), "SELECT\n" + @@ -344,7 +353,7 @@ public void testRewriteFunctionCalls() "SELECT\n" + " TRIM(ARBITRARY(b))\n" + "FROM test_table", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " TRIM(MIN(b))\n" + "FROM test_table"); @@ -355,7 +364,7 @@ public void testRewriteFunctionCalls() "SELECT\n" + " MAP_AGG(a,b)\n" + "FROM test_table", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " TRANSFORM_VALUES(MULTIMAP_AGG(a,b),(k,v)->ARRAY_MAX(v))\n" + "FROM test_table"); @@ -364,7 +373,7 @@ public void testRewriteFunctionCalls() assertCreateTableAs( queryRewriter.rewriteQuery( "SELECT RAND()", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT 1"); // Test rewriting with if expression. @@ -373,7 +382,7 @@ public void testRewriteFunctionCalls() "SELECT\n" + " ARRAY_AGG(DISTINCT a)\n" + "FROM test_table", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " IF(TYPEOF(ARBITRARY(DISTINCT a))='integer', ARRAY_SORT(ARRAY_AGG(DISTINCT a)), ARRAY_AGG(DISTINCT a))\n" + "FROM test_table"); @@ -384,7 +393,7 @@ public void testRewriteFunctionCalls() "SELECT\n" + " TO_UNIXTIME(CURRENT_TIMESTAMP)\n" + "FROM test_table", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " TO_UNIXTIME(TIMESTAMP '2023-01-01 00:00:00 UTC')\n" + "FROM test_table"); @@ -395,7 +404,7 @@ public void testRewriteFunctionCalls() "SELECT\n" + " TO_UNIXTIME(NOW())\n" + "FROM test_table", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " TO_UNIXTIME(DATE_TRUNC('day',NOW()))\n" + "FROM test_table"); @@ -414,7 +423,7 @@ public void testRewriteFunctionCalls() " 1\n" + ") y\n" + " ON (x.b = y.b)", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT *\n" + "FROM test_table x\n" + "JOIN (\n" + @@ -437,7 +446,7 @@ public void testRewriteFunctionCalls() " ARBITRARY(a)\n" + " FROM test_table\n" + ")", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT a, b\n" + "FROM test_table\n" + "WHERE a IN (\n" + @@ -455,7 +464,7 @@ public void testRewriteFunctionCalls() " APPROX_DISTINCT(b) AS num\n" + " FROM test_table\n" + ") x", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT num\n" + "FROM (\n" + " SELECT\n" + @@ -473,7 +482,7 @@ public void testRewriteFunctionCalls() ")\n" + "SELECT\n" + " a\n" + - "FROM x", CONTROL).getQuery(), + "FROM x", CONFIGURATION, CONTROL).getQuery(), "WITH x AS (\n" + " SELECT\n" + " MAX(a) AS a\n" + @@ -499,7 +508,7 @@ public void testRewriteFunctionCalls() " MIN_BY(a, b) AS a\n" + " FROM test_table\n" + ") x", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " MIN(a)\n" + "FROM (\n" + @@ -522,7 +531,7 @@ public void testRewriteFunctionCalls() " PARTITION BY b\n" + " )\n" + "FROM test_table", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " IF(\n" + " MIN(a) OVER (\n" + @@ -549,7 +558,7 @@ public void testRewriteFunctionCalls() " ORDER BY b DESC\n" + " )\n" + "FROM test_table", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " ROW_NUMBER() OVER (\n" + " PARTITION BY b\n" + @@ -564,26 +573,61 @@ public void testRewriteFunctionCalls() " APPROX_PERCENTILE(a, 0.95),\n" + " APPROX_PERCENTILE(a, ARRAY[0.5, 0.9])\n" + "FROM test_table", - CONTROL).getQuery(), + CONFIGURATION, CONTROL).getQuery(), "SELECT\n" + " AVG(a),\n" + " REPEAT(AVG(a), CAST(CARDINALITY(ARRAY[0.5, 0.9]) AS INTEGER))\n" + "FROM test_table"); } + @Test + public void testReuseTableRewrite() + { + String query = "INSERT INTO dest_table SELECT * FROM test_table"; + QueryConfiguration configuration = new QueryConfiguration( + CATALOG, + SCHEMA, + Optional.of("user"), + Optional.empty(), + Optional.empty(), + true); + assertShadowed( + getQueryRewriter(new QueryRewriteConfig().setReuseTable(true), VERIFIER_CONFIG), + query, + "local.tmp", + ImmutableMap.of(TEST, configuration, CONTROL, configuration), + ImmutableList.of(), + query, + ImmutableList.of()); + } + + private void assertShadowed( + QueryRewriter queryRewriter, + @Language("SQL") String query, + String prefix, + List expectedSetupTemplates, + @Language("SQL") String expectedTemplates, + List expectedTeardownTemplates) + { + assertShadowed(queryRewriter, query, prefix, ImmutableMap.of(CONTROL, CONFIGURATION, TEST, CONFIGURATION), expectedSetupTemplates, expectedTemplates, expectedTeardownTemplates); + } + private void assertShadowed( QueryRewriter queryRewriter, @Language("SQL") String query, String prefix, + Map queryConfigurations, List expectedSetupTemplates, @Language("SQL") String expectedTemplates, List expectedTeardownTemplates) { for (ClusterType cluster : ClusterType.values()) { - QueryObjectBundle bundle = queryRewriter.rewriteQuery(query, cluster); + QueryObjectBundle bundle = queryRewriter.rewriteQuery(query, queryConfigurations.get(cluster), cluster, true); String tableName = bundle.getObjectName().toString(); - assertTrue(tableName.startsWith(prefix + "_")); + if (!bundle.isReuseTable()) { + assertTrue(tableName.startsWith(prefix + "_")); + } assertStatements(bundle.getSetupQueries(), templateToStatements(expectedSetupTemplates, tableName)); assertStatements(ImmutableList.of(bundle.getQuery()), templateToStatements(ImmutableList.of(expectedTemplates), tableName)); @@ -593,7 +637,7 @@ private void assertShadowed( private static void assertTableName(QueryRewriter queryRewriter, @Language("SQL") String query, String expectedPrefix) { - QueryObjectBundle bundle = queryRewriter.rewriteQuery(query, CONTROL); + QueryObjectBundle bundle = queryRewriter.rewriteQuery(query, CONFIGURATION, CONTROL); assertTrue(bundle.getObjectName().toString().startsWith(expectedPrefix)); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java index 1a5c23cabc2e5..3127831975eff 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java @@ -59,25 +59,30 @@ public class TestPrestoQuerySourceQuerySupplier " 'schema' control_schema,\n" + " 'user' control_username,\n" + " '{\"a\": \"b\"}' control_session_properties,\n" + + " '[\"x\"]' control_client_tags,\n" + + " query_id control_query_id,\n" + " NULL control_password,\n" + " query test_query,\n" + " 'catalog' test_catalog,\n" + " 'schema' test_schema,\n" + " 'user' test_username,\n" + " NULL test_password,\n" + - " '{\"c\": \"d\"}' test_session_properties\n" + + " '{\"c\": \"d\"}' test_session_properties,\n" + + " '[\"y\"]' test_client_tags,\n" + + " query_id test_query_id\n" + "FROM (\n" + " VALUES\n" + - " ('Q1', 'SELECT 1'),\n" + - " ('Q2', 'INSERT INTO test_table SELECT 1')\n" + - ") queries(name, query)"; + " ('Q1', 'SELECT 1', 'T1'),\n" + + " ('Q2', 'INSERT INTO test_table SELECT 1', 'T2')\n" + + ") queries(name, query, query_id)"; private static final QueryConfiguration CONTROL_CONFIGURATION = new QueryConfiguration( - "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("a", "b"))); + "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("a", "b")), Optional.of(ImmutableList.of("x"))); private static final QueryConfiguration TEST_CONFIGURATION = new QueryConfiguration( - "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("c", "d"))); + "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("c", "d")), Optional.of(ImmutableList.of("y"))); private static final List SOURCE_QUERIES = ImmutableList.of( - new SourceQuery("test", "Q1", "SELECT 1", "SELECT 1", CONTROL_CONFIGURATION, TEST_CONFIGURATION), - new SourceQuery("test", "Q2", "INSERT INTO test_table SELECT 1", "INSERT INTO test_table SELECT 1", CONTROL_CONFIGURATION, TEST_CONFIGURATION)); + new SourceQuery("test", "Q1", "SELECT 1", "SELECT 1", Optional.of("T1"), Optional.of("T1"), CONTROL_CONFIGURATION, TEST_CONFIGURATION), + new SourceQuery("test", "Q2", "INSERT INTO test_table SELECT 1", "INSERT INTO test_table SELECT 1", Optional.of("T2"), Optional.of("T2"), CONTROL_CONFIGURATION, + TEST_CONFIGURATION)); private static StandaloneQueryRunner queryRunner; private static Injector injector;