diff --git a/presto-docs/src/main/sphinx/installation/verifier.rst b/presto-docs/src/main/sphinx/installation/verifier.rst index ddc444c7bd9b8..037324a4e55cd 100644 --- a/presto-docs/src/main/sphinx/installation/verifier.rst +++ b/presto-docs/src/main/sphinx/installation/verifier.rst @@ -20,14 +20,15 @@ Create a MySQL database with the following table and load it with the queries yo control_catalog varchar(256) NOT NULL, control_schema varchar(256) NOT NULL, control_query text NOT NULL, + control_username varchar(256) DEFAULT NULL, + control_password varchar(256) DEFAULT NULL, + control_session_properties text DEFAULT NULL, test_catalog varchar(256) NOT NULL, test_schema varchar(256) NOT NULL, test_query text NOT NULL, - control_username varchar(256) NOT NULL DEFAULT 'verifier-test', - control_password varchar(256) DEFAULT NULL, - test_username varchar(256) NOT NULL DEFAULT 'verifier-test', + test_username varchar(256) DEFAULT NULL, test_password varchar(256) DEFAULT NULL, - session_properties_json varchar(2048) DEFAULT NULL) + test_session_properties text DEFAULT NULL) Next, create a properties file to configure the verifier: @@ -50,32 +51,36 @@ make it executable with ``chmod +x``, then run it: Configuration Reference ----------------------- -================================= ======================================================================= -Name Description -================================= ======================================================================= -``control.timeout`` The maximum execution time of the control queries. -``test.timeout`` The maximum execution time of the test queries. -``metadata.timeout`` The maximum execution time of the queries that are required for - obtaining table metadata or rewriting queries. -``checksum.timeout`` The maximum execution time of the queries that computes checksum for - the control and the test results. -``whitelist`` A comma-separated list that specifies names of the queries within the - suite to verify. -``blacklist`` A comma-separated list that specifies names of the queries to be - excluded from suite. ``blacklist`` is applied after ``whitelist``. -``source-query.table-name`` Specifies the MySQL table from which to read the source queries for - verification. -``event-clients`` A comma-separated list that specifies where the output events should be - emitted. Valid individual values are ``json`` and ``human-readable``. -``json.log-file`` Specifies the output files for JSON events. If ``json`` is specified in - ``event-clients`` but this property is not set, JSON events are emitted - to ``stdout``. -``human-readable.log-file`` Specifies the output files for human readable events. If - ``human-readable`` is specified in ``event-clients`` but this property - is not set, human readable events are emitted to ``stdout``. -``max-concurrency`` Specifies the maximum concurrent verification. Alternatively speaking, - the maximum concurrent queries that will be submitted to control and - test clusters combined. -``relative-error-margin`` Specified the maximum tolerable relative error between control and test - queries for floating point columns. -================================= ======================================================================= +=========================================== ======================================================================= +Name Description +=========================================== ======================================================================= +``control.timeout`` The maximum execution time of the control queries. +``test.timeout`` The maximum execution time of the test queries. +``metadata.timeout`` The maximum execution time of the queries that are required for + obtaining table metadata or rewriting queries. +``checksum.timeout`` The maximum execution time of the queries that computes checksum for + the control and the test results. +``whitelist`` A comma-separated list that specifies names of the queries within the + suite to verify. +``blacklist`` A comma-separated list that specifies names of the queries to be + excluded from suite. ``blacklist`` is applied after ``whitelist``. +``source-query.table-name`` Specifies the MySQL table from which to read the source queries for + verification. +``event-clients`` A comma-separated list that specifies where the output events should be + emitted. Valid individual values are ``json`` and ``human-readable``. +``json.log-file`` Specifies the output files for JSON events. If ``json`` is specified in + ``event-clients`` but this property is not set, JSON events are emitted + to ``stdout``. +``human-readable.log-file`` Specifies the output files for human readable events. If + ``human-readable`` is specified in ``event-clients`` but this property + is not set, human readable events are emitted to ``stdout``. +``max-concurrency`` Specifies the maximum concurrent verification. Alternatively speaking, + the maximum concurrent queries that will be submitted to control and + test clusters combined. +``relative-error-margin`` Specified the maximum tolerable relative error between control and test + queries for floating point columns. +``max-determinism-analysis-runs`` Maximum number of reruns of the control queries in case of a result + mismatch to determine whether the query is deterministic. +``run-teardown-for-determinism-analysis`` Whether temporary tables created in determinism analysis runs are + teared down. +=========================================== ======================================================================= diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 073e9f1bd563e..1f0a67605c83b 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -69,6 +69,11 @@ presto-thrift-connector + + com.fasterxml.jackson.core + jackson-annotations + + com.fasterxml.jackson.core jackson-core diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisDetails.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisDetails.java new file mode 100644 index 0000000000000..77b4f3fcf6731 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisDetails.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.event; + +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventType; +import com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.collect.ImmutableList; + +import javax.annotation.concurrent.Immutable; + +import java.util.List; +import java.util.Optional; + +@Immutable +@EventType("DeterminismAnalysisDetails") +public class DeterminismAnalysisDetails +{ + private final List runs; + private final String limitQueryAnalysis; + private final String limitQueryAnalysisQueryId; + + @JsonCreator + public DeterminismAnalysisDetails( + List runs, + LimitQueryDeterminismAnalysis limitQueryAnalysis, + Optional limitQueryAnalysisQueryId) + { + this.runs = ImmutableList.copyOf(runs); + this.limitQueryAnalysis = limitQueryAnalysis.name(); + this.limitQueryAnalysisQueryId = limitQueryAnalysisQueryId.orElse(null); + } + + @EventField + public List getRuns() + { + return runs; + } + + @EventField + public String getLimitQueryAnalysis() + { + return limitQueryAnalysis; + } + + @EventField + public String getLimitQueryAnalysisQueryId() + { + return limitQueryAnalysisQueryId; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisRun.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisRun.java new file mode 100644 index 0000000000000..9f60cfa6e51ca --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisRun.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.event; + +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventType; + +import javax.annotation.concurrent.Immutable; + +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +@Immutable +@EventType("DeterminismAnalysisRun") +public class DeterminismAnalysisRun +{ + private final String tableName; + private final String queryId; + private final String checksumQueryId; + + private DeterminismAnalysisRun( + Optional tableName, + Optional queryId, + Optional checksumQueryId) + { + this.tableName = tableName.orElse(null); + this.queryId = queryId.orElse(null); + this.checksumQueryId = checksumQueryId.orElse(null); + } + + @EventField + public String getTableName() + { + return tableName; + } + + @EventField + public String getQueryId() + { + return queryId; + } + + @EventField + public String getChecksumQueryId() + { + return checksumQueryId; + } + + public static Builder builder() + { + return new Builder(); + } + + public static class Builder + { + private String tableName; + private String queryId; + private String checksumQueryId; + + private Builder() + { + } + + public Builder setTableName(String tableName) + { + checkState(this.tableName == null, "tableName is already set"); + this.tableName = requireNonNull(tableName, "tableName is null"); + return this; + } + + public Builder setQueryId(String queryId) + { + checkState(this.queryId == null, "queryId is already set"); + this.queryId = requireNonNull(queryId, "queryId is null"); + return this; + } + + public Builder setChecksumQueryId(String checksumQueryId) + { + checkState(this.checksumQueryId == null, "checksumQueryId is already set"); + this.checksumQueryId = requireNonNull(checksumQueryId, "checksumQueryId is null"); + return this; + } + + public DeterminismAnalysisRun build() + { + return new DeterminismAnalysisRun(Optional.ofNullable(tableName), Optional.ofNullable(queryId), Optional.ofNullable(checksumQueryId)); + } + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java index 4358ac27dea53..fe86e1ca4d6b0 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java @@ -48,6 +48,7 @@ public enum EventStatus private final Boolean deterministic; private final String determinismAnalysis; + private final DeterminismAnalysisDetails determinismAnalysisDetails; private final String resolveMessage; private final QueryInfo controlQueryInfo; @@ -66,6 +67,7 @@ public VerifierQueryEvent( EventStatus status, Optional skippedReason, Optional determinismAnalysis, + Optional determinismAnalysisDetails, Optional resolveMessage, Optional controlQueryInfo, Optional testQueryInfo, @@ -81,6 +83,7 @@ public VerifierQueryEvent( this.skippedReason = skippedReason.map(SkippedReason::name).orElse(null); this.deterministic = determinismAnalysis.filter(d -> !d.isUnknown()).map(DeterminismAnalysis::isDeterministic).orElse(null); this.determinismAnalysis = determinismAnalysis.map(DeterminismAnalysis::name).orElse(null); + this.determinismAnalysisDetails = determinismAnalysisDetails.orElse(null); this.resolveMessage = resolveMessage.orElse(null); this.controlQueryInfo = controlQueryInfo.orElse(null); this.testQueryInfo = testQueryInfo.orElse(null); @@ -109,6 +112,7 @@ public static VerifierQueryEvent skipped( Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), ImmutableList.of()); } @@ -155,6 +159,12 @@ public String getDeterminismAnalysis() return determinismAnalysis; } + @EventField + public DeterminismAnalysisDetails getDeterminismAnalysisDetails() + { + return determinismAnalysisDetails; + } + @EventField public String getResolveMessage() { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java index efb789ff8aaf5..5844dc7cb7f99 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java @@ -18,6 +18,7 @@ import com.facebook.presto.sql.SqlFormatter; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.verifier.checksum.ChecksumResult; +import com.facebook.presto.verifier.event.DeterminismAnalysisDetails; import com.facebook.presto.verifier.event.QueryInfo; import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; @@ -91,10 +92,10 @@ public AbstractVerification( this.verificationContext = requireNonNull(verificationContext, "verificationContext is null"); this.testId = requireNonNull(verifierConfig.getTestId(), "testId is null"); - this.runTearDownOnResultMismatch = verifierConfig.isRunTearDownOnResultMismatch(); + this.runTearDownOnResultMismatch = verifierConfig.isRunTeardownOnResultMismatch(); } - protected abstract VerificationResult verify(QueryBundle control, QueryBundle test); + protected abstract MatchResult verify(QueryBundle control, QueryBundle test); protected abstract DeterminismAnalysis analyzeDeterminism(QueryBundle control, ChecksumResult firstChecksum); @@ -110,7 +111,7 @@ public Optional run() boolean resultMismatched = false; QueryBundle control = null; QueryBundle test = null; - VerificationResult verificationResult = null; + MatchResult matchResult = null; Optional determinismAnalysis = Optional.empty(); QueryStats controlQueryStats = null; @@ -121,15 +122,15 @@ public Optional run() test = queryRewriter.rewriteQuery(sourceQuery.getTestQuery(), TEST); controlQueryStats = setupAndRun(control, false); testQueryStats = setupAndRun(test, false); - verificationResult = verify(control, test); + matchResult = verify(control, test); - if (verificationResult.getMatchResult().isMismatchPossiblyCausedByNonDeterminism()) { - determinismAnalysis = Optional.of(analyzeDeterminism(control, verificationResult.getMatchResult().getControlChecksum())); + if (matchResult.isMismatchPossiblyCausedByNonDeterminism()) { + determinismAnalysis = Optional.of(analyzeDeterminism(control, matchResult.getControlChecksum())); } boolean maybeDeterministic = !determinismAnalysis.isPresent() || determinismAnalysis.get().isDeterministic() || determinismAnalysis.get().isUnknown(); - resultMismatched = maybeDeterministic && !verificationResult.getMatchResult().isMatched(); + resultMismatched = maybeDeterministic && !matchResult.isMatched(); return Optional.of(buildEvent( Optional.of(control), @@ -137,7 +138,7 @@ public Optional run() Optional.ofNullable(controlQueryStats), Optional.ofNullable(testQueryStats), Optional.empty(), - Optional.of(verificationResult), + Optional.of(matchResult), determinismAnalysis)); } catch (QueryException e) { @@ -150,7 +151,7 @@ public Optional run() Optional.ofNullable(controlQueryStats), Optional.ofNullable(testQueryStats), Optional.of(e), - Optional.ofNullable(verificationResult), + Optional.ofNullable(matchResult), determinismAnalysis)); } catch (Throwable t) { @@ -175,6 +176,11 @@ protected QueryRewriter getQueryRewriter() return queryRewriter; } + protected VerificationContext getVerificationContext() + { + return verificationContext; + } + protected QueryStats setupAndRun(QueryBundle bundle, boolean determinismAnalysis) { checkState(!determinismAnalysis || bundle.getCluster() == CONTROL, "Determinism analysis can only be run on control cluster"); @@ -209,10 +215,10 @@ private VerifierQueryEvent buildEvent( Optional controlStats, Optional testStats, Optional queryException, - Optional verificationResult, + Optional matchResult, Optional determinismAnalysis) { - boolean succeeded = verificationResult.isPresent() && verificationResult.get().getMatchResult().isMatched(); + boolean succeeded = matchResult.isPresent() && matchResult.get().isMatched(); QueryState controlState = getQueryState(controlStats, queryException, CONTROL); QueryState testState = getQueryState(testStats, queryException, TEST); @@ -227,8 +233,8 @@ private VerifierQueryEvent buildEvent( queryException.get().getQueryStage().getTargetCluster(), getStackTraceAsString(queryException.get().getCause())); } - if (verificationResult.isPresent()) { - errorMessage += verificationResult.get().getMatchResult().getResultsComparison(); + if (matchResult.isPresent()) { + errorMessage += matchResult.get().getResultsComparison(); } } @@ -259,7 +265,7 @@ else if (skippedReason.isPresent()) { Optional errorCode = Optional.empty(); if (!succeeded) { errorCode = Optional.ofNullable(queryException.map(QueryException::getErrorCode).orElse( - verificationResult.map(VerificationResult::getMatchResult).map(MatchResult::getMatchType).map(MatchType::name).orElse(null))); + matchResult.map(MatchResult::getMatchType).map(MatchType::name).orElse(null))); } return new VerifierQueryEvent( @@ -269,19 +275,25 @@ else if (skippedReason.isPresent()) { status, skippedReason, determinismAnalysis, + determinismAnalysis.isPresent() ? + Optional.of(new DeterminismAnalysisDetails( + verificationContext.getDeterminismAnalysisRuns(), + verificationContext.getLimitQueryAnalysis(), + verificationContext.getLimitQueryAnalysisQueryId())) : + Optional.empty(), resolveMessage, Optional.of(buildQueryInfo( sourceQuery.getControlConfiguration(), sourceQuery.getControlQuery(), - verificationResult.map(VerificationResult::getControlChecksumQueryId), - verificationResult.map(VerificationResult::getControlChecksumQuery), + verificationContext.getControlChecksumQueryId(), + verificationContext.getControlChecksumQuery(), control, controlStats)), Optional.of(buildQueryInfo( sourceQuery.getTestConfiguration(), sourceQuery.getTestQuery(), - verificationResult.map(VerificationResult::getTestChecksumQueryId), - verificationResult.map(VerificationResult::getTestChecksumQuery), + verificationContext.getTestChecksumQueryId(), + verificationContext.getTestChecksumQuery(), test, testStats)), errorCode, diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java index 3059bf28336a4..5aea0f1d289c8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java @@ -20,12 +20,14 @@ import com.facebook.presto.verifier.checksum.ChecksumResult; import com.facebook.presto.verifier.checksum.ChecksumValidator; import com.facebook.presto.verifier.checksum.ColumnMatchResult; +import com.facebook.presto.verifier.event.DeterminismAnalysisRun; import com.facebook.presto.verifier.framework.MatchResult.MatchType; import com.facebook.presto.verifier.prestoaction.PrestoAction; import com.facebook.presto.verifier.resolver.FailureResolverManager; import com.facebook.presto.verifier.rewrite.QueryRewriter; import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -46,6 +48,8 @@ import static com.facebook.presto.verifier.framework.MatchResult.MatchType.SCHEMA_MISMATCH; import static com.facebook.presto.verifier.framework.QueryStage.CHECKSUM; import static com.facebook.presto.verifier.framework.QueryStage.DESCRIBE; +import static com.facebook.presto.verifier.framework.VerifierUtil.callWithQueryStatsConsumer; +import static com.facebook.presto.verifier.framework.VerifierUtil.runWithQueryStatsConsumer; import static com.google.common.collect.Iterables.getOnlyElement; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -56,6 +60,9 @@ public class DataVerification private final TypeManager typeManager; private final ChecksumValidator checksumValidator; private final LimitQueryDeterminismAnalyzer limitQueryDeterminismAnalyzer; + private final boolean runTeardownForDeterminismAnalysis; + + private final int maxDeterminismAnalysisRuns; public DataVerification( VerificationResubmitter verificationResubmitter, @@ -73,50 +80,60 @@ public DataVerification( this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.checksumValidator = requireNonNull(checksumValidator, "checksumValidator is null"); this.limitQueryDeterminismAnalyzer = requireNonNull(limitQueryDeterminismAnalyzer, "limitQueryDeterminismAnalyzer is null"); + this.runTeardownForDeterminismAnalysis = verifierConfig.isRunTeardownForDeterminismAnalysis(); + this.maxDeterminismAnalysisRuns = verifierConfig.getMaxDeterminismAnalysisRuns(); } @Override - public VerificationResult verify(QueryBundle control, QueryBundle test) + public MatchResult verify(QueryBundle control, QueryBundle test) { List controlColumns = getColumns(control.getTableName()); List testColumns = getColumns(test.getTableName()); - ChecksumQueryAndResult controlChecksum = computeChecksum(control, controlColumns); - ChecksumQueryAndResult testChecksum = computeChecksum(test, testColumns); - return new VerificationResult( - controlChecksum.getQueryId(), - testChecksum.getQueryId(), - formatSql(controlChecksum.getQuery()), - formatSql(testChecksum.getQuery()), - match( - controlColumns, - testColumns, - controlChecksum.getResult(), - testChecksum.getResult())); + + Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getTableName(), controlColumns); + Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getTableName(), testColumns); + + getVerificationContext().setControlChecksumQuery(formatSql(controlChecksumQuery)); + getVerificationContext().setTestChecksumQuery(formatSql(testChecksumQuery)); + + QueryResult controlChecksum = callWithQueryStatsConsumer( + () -> executeChecksumQuery(controlChecksumQuery), + stats -> getVerificationContext().setControlChecksumQueryId(stats.getQueryId())); + QueryResult testChecksum = callWithQueryStatsConsumer( + () -> executeChecksumQuery(testChecksumQuery), + stats -> getVerificationContext().setTestChecksumQueryId(stats.getQueryId())); + + return match(controlColumns, testColumns, getOnlyElement(controlChecksum.getResults()), getOnlyElement(testChecksum.getResults())); } @Override - protected DeterminismAnalysis analyzeDeterminism(QueryBundle control, ChecksumResult firstChecksum) + protected DeterminismAnalysis analyzeDeterminism(QueryBundle control, ChecksumResult controlChecksum) { List columns = getColumns(control.getTableName()); + List queryBundles = new ArrayList<>(); - QueryBundle secondRun = null; - QueryBundle thirdRun = null; try { - secondRun = getQueryRewriter().rewriteQuery(getSourceQuery().getControlQuery(), CONTROL); - setupAndRun(secondRun, true); - DeterminismAnalysis determinismAnalysis = matchResultToDeterminism(match(columns, columns, firstChecksum, computeChecksum(secondRun, columns).getResult())); - if (determinismAnalysis != DETERMINISTIC) { - return determinismAnalysis; + for (int i = 0; i < maxDeterminismAnalysisRuns; i++) { + QueryBundle queryBundle = getQueryRewriter().rewriteQuery(getSourceQuery().getControlQuery(), CONTROL); + queryBundles.add(queryBundle); + DeterminismAnalysisRun.Builder run = getVerificationContext().startDeterminismAnalysisRun().setTableName(queryBundle.getTableName().toString()); + + runWithQueryStatsConsumer(() -> setupAndRun(queryBundle, true), stats -> run.setQueryId(stats.getQueryId())); + + Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getTableName(), columns); + ChecksumResult testChecksum = getOnlyElement(callWithQueryStatsConsumer( + () -> executeChecksumQuery(checksumQuery), + stats -> run.setChecksumQueryId(stats.getQueryId())).getResults()); + + DeterminismAnalysis determinismAnalysis = matchResultToDeterminism(match(columns, columns, controlChecksum, testChecksum)); + if (determinismAnalysis != DETERMINISTIC) { + return determinismAnalysis; + } } - thirdRun = getQueryRewriter().rewriteQuery(getSourceQuery().getControlQuery(), CONTROL); - setupAndRun(thirdRun, true); - determinismAnalysis = matchResultToDeterminism(match(columns, columns, firstChecksum, computeChecksum(thirdRun, columns).getResult())); - if (determinismAnalysis != DETERMINISTIC) { - return determinismAnalysis; - } + LimitQueryDeterminismAnalysis analysis = limitQueryDeterminismAnalyzer.analyze(control, controlChecksum.getRowCount(), getVerificationContext()); + getVerificationContext().setLimitQueryAnalysis(analysis); - LimitQueryDeterminismAnalyzer.Analysis analysis = limitQueryDeterminismAnalyzer.analyze(control, firstChecksum.getRowCount()); switch (analysis) { case NON_DETERMINISTIC: return NON_DETERMINISTIC_LIMIT_CLAUSE; @@ -136,8 +153,9 @@ protected DeterminismAnalysis analyzeDeterminism(QueryBundle control, ChecksumRe return ANALYSIS_FAILED; } finally { - teardownSafely(secondRun); - teardownSafely(thirdRun); + if (runTeardownForDeterminismAnalysis) { + queryBundles.forEach(this::teardownSafely); + } } } @@ -200,45 +218,8 @@ private List getColumns(QualifiedName tableName) .getResults(); } - private ChecksumQueryAndResult computeChecksum(QueryBundle bundle, List columns) + private QueryResult executeChecksumQuery(Query query) { - Query checksumQuery = checksumValidator.generateChecksumQuery(bundle.getTableName(), columns); - QueryResult queryResult = getPrestoAction().execute( - checksumQuery, - CHECKSUM, - ChecksumResult::fromResultSet); - return new ChecksumQueryAndResult( - queryResult.getQueryStats().getQueryId(), - checksumQuery, - getOnlyElement(queryResult.getResults())); - } - - private class ChecksumQueryAndResult - { - private final String queryId; - private final Query query; - private final ChecksumResult result; - - public ChecksumQueryAndResult(String queryId, Query query, ChecksumResult result) - { - this.queryId = requireNonNull(queryId, "queryId is null"); - this.query = requireNonNull(query, "query is null"); - this.result = requireNonNull(result, "result is null"); - } - - public String getQueryId() - { - return queryId; - } - - public Query getQuery() - { - return query; - } - - public ChecksumResult getResult() - { - return result; - } + return getPrestoAction().execute(query, CHECKSUM, ChecksumResult::fromResultSet); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalysis.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalysis.java new file mode 100644 index 0000000000000..c017ce5d5fe09 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalysis.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +public enum LimitQueryDeterminismAnalysis +{ + NOT_RUN, + NON_DETERMINISTIC, + DETERMINISTIC, + FAILED_DATA_CHANGED, +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalyzer.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalyzer.java index a775ca50c945c..74fdae65a164c 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalyzer.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalyzer.java @@ -11,7 +11,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.facebook.presto.verifier.framework; import com.facebook.presto.sql.tree.CreateTableAsSelect; @@ -32,24 +31,17 @@ import java.util.Optional; import static com.facebook.presto.sql.QueryUtil.simpleQuery; -import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis.DETERMINISTIC; -import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis.FAILED_DATA_CHANGED; -import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis.NON_DETERMINISTIC; -import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis.NOT_RUN; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.DETERMINISTIC; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.FAILED_DATA_CHANGED; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NON_DETERMINISTIC; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NOT_RUN; import static com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS; +import static com.facebook.presto.verifier.framework.VerifierUtil.callWithQueryStatsConsumer; import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Objects.requireNonNull; public class LimitQueryDeterminismAnalyzer { - public enum Analysis - { - NOT_RUN, - NON_DETERMINISTIC, - DETERMINISTIC, - FAILED_DATA_CHANGED, - } - private final PrestoAction prestoAction; private final boolean enabled; @@ -59,7 +51,7 @@ public LimitQueryDeterminismAnalyzer(PrestoAction prestoAction, VerifierConfig v this.enabled = verifierConfig.isEnableLimitQueryDeterminismAnalyzer(); } - public Analysis analyze(QueryBundle control, long rowCount) + public LimitQueryDeterminismAnalysis analyze(QueryBundle control, long rowCount, VerificationContext verificationContext) { if (!enabled) { return NOT_RUN; @@ -130,8 +122,12 @@ else if (query.getQueryBody() instanceof QuerySpecification) { Query rowCountQuery = simpleQuery( new Select(false, ImmutableList.of(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new LongLiteral("1")))))), new TableSubquery(queryNoLimit)); - long rowCountNoLimit = getOnlyElement(prestoAction.execute(rowCountQuery, DETERMINISM_ANALYSIS, resultSet -> resultSet.getLong(1)).getResults()); + QueryResult result = callWithQueryStatsConsumer( + () -> prestoAction.execute(rowCountQuery, DETERMINISM_ANALYSIS, resultSet -> resultSet.getLong(1)), + stats -> verificationContext.setLimitQueryAnalysisQueryId(stats.getQueryId())); + + long rowCountNoLimit = getOnlyElement(result.getResults()); if (rowCountNoLimit > rowCount) { return NON_DETERMINISTIC; } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryBundle.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryBundle.java index ef4a1e8835003..228f299bb56c7 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryBundle.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryBundle.java @@ -18,21 +18,19 @@ import com.google.common.collect.ImmutableList; import java.util.List; -import java.util.Optional; -import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; public class QueryBundle { - private final Optional tableName; + private final QualifiedName tableName; private final List setupQueries; private final Statement query; private final List teardownQueries; private final ClusterType cluster; public QueryBundle( - Optional tableName, + QualifiedName tableName, List setupQueries, Statement query, List teardownQueries, @@ -47,8 +45,7 @@ public QueryBundle( public QualifiedName getTableName() { - checkState(tableName.isPresent(), "tableName is missing"); - return tableName.get(); + return tableName; } public List getSetupQueries() diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationContext.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationContext.java index dea839d791968..a32d817cf002f 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationContext.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationContext.java @@ -13,17 +13,112 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.verifier.event.DeterminismAnalysisRun; import com.facebook.presto.verifier.event.QueryFailure; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Optional; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NOT_RUN; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; public class VerificationContext { + private String controlChecksumQueryId; + private String controlChecksumQuery; + private String testChecksumQueryId; + private String testChecksumQuery; + + private ImmutableList.Builder determinismAnalysisRuns = ImmutableList.builder(); + private LimitQueryDeterminismAnalysis limitQueryAnalysis; + private String limitQueryAnalysisQueryId; + private ImmutableSet.Builder queryExceptions = ImmutableSet.builder(); + public Optional getControlChecksumQueryId() + { + return Optional.ofNullable(controlChecksumQueryId); + } + + public void setControlChecksumQueryId(String controlChecksumQueryId) + { + checkState(this.controlChecksumQueryId == null, "controlChecksumQueryId is already set"); + this.controlChecksumQueryId = requireNonNull(controlChecksumQueryId, "controlChecksumQueryId is null"); + } + + public Optional getControlChecksumQuery() + { + return Optional.ofNullable(controlChecksumQuery); + } + + public void setControlChecksumQuery(String controlChecksumQuery) + { + checkState(this.controlChecksumQuery == null, "controlChecksumQuery is already set"); + this.controlChecksumQuery = requireNonNull(controlChecksumQuery, "controlChecksumQuery is null"); + } + + public Optional getTestChecksumQueryId() + { + return Optional.ofNullable(testChecksumQueryId); + } + + public void setTestChecksumQueryId(String testChecksumQueryId) + { + checkState(this.testChecksumQueryId == null, "testChecksumQueryId is already set"); + this.testChecksumQueryId = requireNonNull(testChecksumQueryId, "testChecksumQueryId is null"); + } + + public Optional getTestChecksumQuery() + { + return Optional.ofNullable(testChecksumQuery); + } + + public void setTestChecksumQuery(String testChecksumQuery) + { + checkState(this.testChecksumQuery == null, "testChecksumQuery is already set"); + this.testChecksumQuery = requireNonNull(testChecksumQuery, "testChecksumQuery is null"); + } + + public List getDeterminismAnalysisRuns() + { + return determinismAnalysisRuns.build().stream() + .map(DeterminismAnalysisRun.Builder::build) + .collect(toImmutableList()); + } + + public DeterminismAnalysisRun.Builder startDeterminismAnalysisRun() + { + DeterminismAnalysisRun.Builder run = DeterminismAnalysisRun.builder(); + determinismAnalysisRuns.add(run); + return run; + } + + public LimitQueryDeterminismAnalysis getLimitQueryAnalysis() + { + return limitQueryAnalysis == null ? NOT_RUN : limitQueryAnalysis; + } + + public void setLimitQueryAnalysis(LimitQueryDeterminismAnalysis limitQueryAnalysis) + { + checkState(this.limitQueryAnalysis == null, "limitQueryAnalysis is already set"); + this.limitQueryAnalysis = requireNonNull(limitQueryAnalysis, "limitQueryAnalysis is null"); + } + + public Optional getLimitQueryAnalysisQueryId() + { + return Optional.ofNullable(limitQueryAnalysisQueryId); + } + + public void setLimitQueryAnalysisQueryId(String limitQueryAnalysisQueryId) + { + checkState(this.limitQueryAnalysisQueryId == null, "limitQueryAnalysisQueryId is already set"); + this.limitQueryAnalysisQueryId = requireNonNull(limitQueryAnalysisQueryId, "limitQueryAnalysisQueryId is null"); + } + public void addException(QueryException exception) { queryExceptions.add(exception); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationResult.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationResult.java deleted file mode 100644 index a8f1ec0402843..0000000000000 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationResult.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.verifier.framework; - -import static java.util.Objects.requireNonNull; - -public class VerificationResult -{ - private final String controlChecksumQueryId; - private final String testChecksumQueryId; - private final String controlChecksumQuery; - private final String testChecksumQuery; - private final MatchResult matchResult; - - public VerificationResult( - String controlChecksumQueryId, - String testChecksumQueryId, - String controlChecksumQuery, - String testChecksumQuery, - MatchResult matchResult) - { - this.controlChecksumQueryId = requireNonNull(controlChecksumQueryId, "controlChecksumQueryId is null"); - this.testChecksumQueryId = requireNonNull(testChecksumQueryId, "testChecksumQueryId is null"); - this.controlChecksumQuery = requireNonNull(controlChecksumQuery, "controlChecksumQuery is null"); - this.testChecksumQuery = requireNonNull(testChecksumQuery, "testChecksumQuery is null"); - this.matchResult = requireNonNull(matchResult, "matchResult is null"); - } - - public String getControlChecksumQueryId() - { - return controlChecksumQueryId; - } - - public String getControlChecksumQuery() - { - return controlChecksumQuery; - } - - public String getTestChecksumQueryId() - { - return testChecksumQueryId; - } - - public String getTestChecksumQuery() - { - return testChecksumQuery; - } - - public MatchResult getMatchResult() - { - return matchResult; - } -} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java index 8e29563c57c62..dbdf05ec60d14 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java @@ -44,7 +44,10 @@ public class VerifierConfig private double relativeErrorMargin = 1e-4; private double absoluteErrorMargin = 1e-12; - private boolean runTearDownOnResultMismatch; + private boolean runTeardownOnResultMismatch; + private boolean runTeardownForDeterminismAnalysis; + + private int maxDeterminismAnalysisRuns = 2; private boolean enableLimitQueryDeterminismAnalyzer = true; private int verificationResubmissionLimit = 2; @@ -219,16 +222,42 @@ public VerifierConfig setAbsoluteErrorMargin(double absoluteErrorMargin) return this; } - public boolean isRunTearDownOnResultMismatch() + public boolean isRunTeardownOnResultMismatch() { - return runTearDownOnResultMismatch; + return runTeardownOnResultMismatch; } @ConfigDescription("When set to false, temporary tables are not dropped in case of checksum failure") @Config("run-teardown-on-result-mismatch") - public VerifierConfig setRunTearDownOnResultMismatch(boolean runTearDownOnResultMismatch) + public VerifierConfig setRunTeardownOnResultMismatch(boolean runTeardownOnResultMismatch) + { + this.runTeardownOnResultMismatch = runTeardownOnResultMismatch; + return this; + } + + public boolean isRunTeardownForDeterminismAnalysis() + { + return runTeardownForDeterminismAnalysis; + } + + @ConfigDescription("When set to false, temporary tables are not dropped for determinism analysis runs") + @Config("run-teardown-for-determinism-analysis") + public VerifierConfig setRunTeardownForDeterminismAnalysis(boolean runTeardownForDeterminismAnalysis) + { + this.runTeardownForDeterminismAnalysis = runTeardownForDeterminismAnalysis; + return this; + } + + @Min(0) + public int getMaxDeterminismAnalysisRuns() + { + return maxDeterminismAnalysisRuns; + } + + @Config("max-determinism-analysis-runs") + public VerifierConfig setMaxDeterminismAnalysisRuns(int maxDeterminismAnalysisRuns) { - this.runTearDownOnResultMismatch = runTearDownOnResultMismatch; + this.maxDeterminismAnalysisRuns = maxDeterminismAnalysisRuns; return this; } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java index 7b2a39904cf86..543063217cbc3 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java @@ -13,10 +13,15 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.tree.Identifier; +import java.util.function.Consumer; +import java.util.function.Function; + import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE; +import static com.google.common.base.Functions.identity; public class VerifierUtil { @@ -30,4 +35,32 @@ public static Identifier delimitedIdentifier(String name) { return new Identifier(name, true); } + + public static void runWithQueryStatsConsumer(Callable callable, Consumer queryStatsConsumer) + { + callWithQueryStatsConsumer(callable, identity(), queryStatsConsumer); + } + + public static QueryResult callWithQueryStatsConsumer(Callable> callable, Consumer queryStatsConsumer) + { + return callWithQueryStatsConsumer(callable, QueryResult::getQueryStats, queryStatsConsumer); + } + + private static V callWithQueryStatsConsumer(Callable callable, Function queryStatsTransformer, Consumer queryStatsConsumer) + { + try { + V result = callable.call(); + queryStatsConsumer.accept(queryStatsTransformer.apply(result)); + return result; + } + catch (QueryException e) { + e.getQueryStats().ifPresent(queryStatsConsumer); + throw e; + } + } + + public interface Callable + { + V call(); + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java index 5201922e69052..5f9a5b658e9db 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java @@ -48,6 +48,7 @@ import static com.facebook.presto.verifier.framework.QueryStage.REWRITE; import static com.facebook.presto.verifier.framework.QueryType.Category.DATA_PRODUCING; import static com.facebook.presto.verifier.framework.VerifierUtil.PARSING_OPTIONS; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; @@ -76,16 +77,15 @@ public QueryBundle rewriteQuery(@Language("SQL") String query, ClusterType clust { checkState(prefixes.containsKey(clusterType), "Unsupported cluster type: %s", clusterType); Statement statement = sqlParser.createStatement(query, PARSING_OPTIONS); - if (QueryType.of(statement).getCategory() != DATA_PRODUCING) { - return new QueryBundle(Optional.empty(), ImmutableList.of(), statement, ImmutableList.of(), clusterType); - } + QueryType queryType = QueryType.of(statement); + checkArgument(queryType.getCategory() == DATA_PRODUCING, "Unsupported statement type: %s", queryType); QualifiedName prefix = prefixes.get(clusterType); if (statement instanceof CreateTableAsSelect) { CreateTableAsSelect createTableAsSelect = (CreateTableAsSelect) statement; QualifiedName temporaryTableName = generateTemporaryTableName(Optional.of(createTableAsSelect.getName()), prefix); return new QueryBundle( - Optional.of(temporaryTableName), + temporaryTableName, ImmutableList.of(), new CreateTableAsSelect( temporaryTableName, @@ -103,7 +103,7 @@ public QueryBundle rewriteQuery(@Language("SQL") String query, ClusterType clust QualifiedName originalTableName = insert.getTarget(); QualifiedName temporaryTableName = generateTemporaryTableName(Optional.of(originalTableName), prefix); return new QueryBundle( - Optional.of(temporaryTableName), + temporaryTableName, ImmutableList.of( new CreateTable( temporaryTableName, @@ -121,7 +121,7 @@ public QueryBundle rewriteQuery(@Language("SQL") String query, ClusterType clust if (statement instanceof Query) { QualifiedName temporaryTableName = generateTemporaryTableName(Optional.empty(), prefix); return new QueryBundle( - Optional.of(temporaryTableName), + temporaryTableName, ImmutableList.of(), new CreateTableAsSelect( temporaryTableName, diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java index 431b2ecfead90..58e4e0bb7bcc5 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java @@ -19,12 +19,31 @@ import org.jdbi.v3.sqlobject.customizer.Bind; import org.jdbi.v3.sqlobject.customizer.Define; import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; import java.util.List; @RegisterColumnMapper(StringToStringMapColumnMapper.class) public interface VerifierDao { + @SqlUpdate("CREATE TABLE verifier_queries (\n" + + " id int(11) unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n" + + " suite varchar(256) NOT NULL,\n" + + " name varchar(256) DEFAULT NULL,\n" + + " control_catalog varchar(256) NOT NULL,\n" + + " control_schema varchar(256) NOT NULL,\n" + + " control_query text NOT NULL,\n" + + " control_username varchar(256) DEFAULT NULL,\n" + + " control_password varchar(256) DEFAULT NULL,\n" + + " control_session_properties text DEFAULT NULL,\n" + + " test_catalog varchar(256) NOT NULL,\n" + + " test_schema varchar(256) NOT NULL,\n" + + " test_query text NOT NULL,\n" + + " test_username varchar(256) DEFAULT NULL,\n" + + " test_password varchar(256) DEFAULT NULL,\n" + + " test_session_properties text DEFAULT NULL)") + void createVerifierQueriesTable(@Define("table_name") String tableName); + @SqlQuery("SELECT\n" + " suite,\n" + " name,\n" + @@ -33,13 +52,13 @@ public interface VerifierDao " control_schema,\n" + " control_username,\n" + " control_password,\n" + - " session_properties_json test_session_properties,\n" + + " control_session_properties,\n" + " test_query,\n" + " test_catalog,\n" + " test_schema,\n" + " test_username,\n" + " test_password,\n" + - " session_properties_json control_session_properties\n" + + " test_session_properties\n" + "FROM\n" + " \n" + "WHERE\n" + diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java index 6fe7f319d0dd4..3381bebda8661 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java @@ -18,6 +18,8 @@ import com.facebook.presto.testing.mysql.MySqlOptions; import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.StandaloneQueryRunner; +import com.facebook.presto.verifier.source.MySqlSourceQueryConfig; +import com.facebook.presto.verifier.source.VerifierDao; import com.google.common.collect.ImmutableList; import io.airlift.units.Duration; import org.jdbi.v3.core.Handle; @@ -55,31 +57,12 @@ public static StandaloneQueryRunner setupPresto() return queryRunner; } - public static String getJdbcUrl(StandaloneQueryRunner queryRunner) - { - return queryRunner.getServer().getBaseUrl().toString().replace("http", "jdbc:presto"); - } - public static TestingMySqlServer setupMySql() throws Exception { TestingMySqlServer mySqlServer = new TestingMySqlServer("testuser", "testpass", ImmutableList.of(XDB), MY_SQL_OPTIONS); try (Handle handle = getHandle(mySqlServer)) { - handle.execute("CREATE TABLE verifier_queries (\n" + - " id int(11) unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n" + - " suite varchar(256) NOT NULL,\n" + - " name varchar(256) DEFAULT NULL,\n" + - " control_catalog varchar(256) NOT NULL,\n" + - " control_schema varchar(256) NOT NULL,\n" + - " control_query text NOT NULL,\n" + - " test_catalog varchar(256) NOT NULL,\n" + - " test_schema varchar(256) NOT NULL,\n" + - " test_query text NOT NULL,\n" + - " control_username varchar(256) NOT NULL DEFAULT 'verifier-test',\n" + - " control_password varchar(256) DEFAULT NULL,\n" + - " test_username varchar(256) NOT NULL DEFAULT 'verifier-test',\n" + - " test_password varchar(256) DEFAULT NULL,\n" + - " session_properties_json varchar(2048) DEFAULT NULL)"); + handle.attach(VerifierDao.class).createVerifierQueriesTable(new MySqlSourceQueryConfig().getTableName()); } return mySqlServer; } @@ -93,7 +76,7 @@ public static void insertSourceQuery(Handle handle, String suite, String name, S { handle.execute( "INSERT INTO verifier_queries(\n" + - " suite, name, control_catalog, control_schema, control_query, test_catalog, test_schema, test_query, control_username, test_username)\n" + + " suite, name, control_catalog, control_schema, control_query, test_catalog, test_schema, test_query)\n" + "SELECT\n" + " ?,\n" + " ?,\n" + @@ -102,9 +85,7 @@ public static void insertSourceQuery(Handle handle, String suite, String name, S " ?,\n" + " 'verifier',\n" + " 'default',\n" + - " ?,\n" + - " 'verifier_test',\n" + - " 'verifier_test'", + " ?", suite, name, query, diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java index d239a41c1b154..e53412d98008e 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java @@ -22,6 +22,7 @@ import com.facebook.presto.verifier.checksum.FloatingPointColumnValidator; import com.facebook.presto.verifier.checksum.OrderableArrayColumnValidator; import com.facebook.presto.verifier.checksum.SimpleColumnValidator; +import com.facebook.presto.verifier.event.DeterminismAnalysisRun; import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; import com.facebook.presto.verifier.prestoaction.JdbcPrestoAction; @@ -31,14 +32,17 @@ import com.facebook.presto.verifier.resolver.FailureResolverManager; import com.facebook.presto.verifier.retry.RetryConfig; import com.facebook.presto.verifier.rewrite.QueryRewriter; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.List; import java.util.Optional; import java.util.regex.Pattern; +import java.util.stream.IntStream; import static com.facebook.presto.sql.parser.IdentifierSymbol.AT_SIGN; import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; @@ -55,10 +59,14 @@ import static com.facebook.presto.verifier.framework.SkippedReason.CONTROL_SETUP_QUERY_FAILED; import static com.facebook.presto.verifier.framework.SkippedReason.FAILED_BEFORE_CONTROL_QUERY; import static com.facebook.presto.verifier.framework.SkippedReason.NON_DETERMINISTIC; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; import static java.util.regex.Pattern.DOTALL; import static java.util.regex.Pattern.MULTILINE; +import static java.util.stream.Collectors.joining; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; @Test(singleThreaded = true) @@ -78,11 +86,15 @@ public void setupClass() } private DataVerification createVerification(String controlQuery, String testQuery) + { + return createVerification(controlQuery, testQuery, new VerifierConfig().setTestId(TEST_ID)); + } + + private DataVerification createVerification(String controlQuery, String testQuery, VerifierConfig verifierConfig) { QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty()); VerificationContext verificationContext = new VerificationContext(); RetryConfig retryConfig = new RetryConfig(); - VerifierConfig verifierConfig = new VerifierConfig().setTestId(TEST_ID); PrestoAction prestoAction = new JdbcPrestoAction( new PrestoExceptionClassifier(ImmutableSet.of(), ImmutableSet.of()), configuration, @@ -224,12 +236,18 @@ public void testNonDeterministic() "Control 1 rows, Test 1 rows\n" + "Mismatched Columns:\n" + " _col0 \\(double\\): control\\(sum: .*\\) test\\(sum: 2.0\\) relative error: .*\n")); + + List runs = event.get().getDeterminismAnalysisDetails().getRuns(); + assertEquals(runs.size(), 1); + assertDeterminismAnalysisRun(runs.get(0)); } @Test public void testArrayOfRow() { - Optional event = createVerification("SELECT ARRAY[ROW(1, 'a'), ROW(2, null)]", "SELECT ARRAY[ROW(1, 'a'), ROW(2, null)]").run(); + Optional event = createVerification( + "SELECT ARRAY[ROW(1, 'a'), ROW(2, null)]", "SELECT ARRAY[ROW(1, 'a'), ROW(2, null)]", + new VerifierConfig().setTestId(TEST_ID).setMaxDeterminismAnalysisRuns(3)).run(); assertTrue(event.isPresent()); assertEvent(event.get(), SUCCEEDED, Optional.empty(), Optional.empty(), Optional.empty()); @@ -245,6 +263,28 @@ public void testArrayOfRow() "Control 1 rows, Test 1 rows\n" + "Mismatched Columns:\n" + " _col0 \\(array\\(row\\(integer, varchar\\(1\\)\\)\\)\\): control\\(checksum: 71 b5 2f 7f 1e 9b a6 a4\\) test\\(checksum: b4 3c 7d 02 2b 14 77 12\\)\n")); + + List runs = event.get().getDeterminismAnalysisDetails().getRuns(); + assertEquals(runs.size(), 2); + assertDeterminismAnalysisRun(runs.get(0)); + assertDeterminismAnalysisRun(runs.get(1)); + } + + @Test + public void testChecksumQueryFailed() + { + List columns = IntStream.range(0, 1000).mapToObj(i -> "c" + i).collect(toImmutableList()); + queryRunner.execute(format("CREATE TABLE checksum_test (%s)", columns.stream().map(column -> column + " double").collect(joining(",")))); + + String query = format("SELECT %s FROM checksum_test", Joiner.on(",").join(columns)); + Optional event = createVerification(query, query).run(); + + assertTrue(event.isPresent()); + assertEquals(event.get().getStatus(), FAILED.name()); + assertEquals(event.get().getErrorCode(), "PRESTO(COMPILER_ERROR)"); + assertNotNull(event.get().getControlQueryInfo().getChecksumQuery()); + assertNotNull(event.get().getControlQueryInfo().getChecksumQueryId()); + assertNotNull(event.get().getTestQueryInfo().getChecksumQuery()); } private void assertEvent( @@ -268,4 +308,11 @@ private void assertEvent( assertTrue(Pattern.compile(expectedErrorMessageRegex.get(), MULTILINE + DOTALL).matcher(event.getErrorMessage()).matches()); } } + + private void assertDeterminismAnalysisRun(DeterminismAnalysisRun run) + { + assertNotNull(run.getTableName()); + assertNotNull(run.getQueryId()); + assertNotNull(run.getChecksumQueryId()); + } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java index 7af32fa916715..ba49006a56f85 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java @@ -19,7 +19,6 @@ import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Statement; -import com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis; import com.facebook.presto.verifier.prestoaction.PrestoAction; import com.google.common.collect.ImmutableList; import org.testng.annotations.BeforeMethod; @@ -33,13 +32,15 @@ import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE; import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; -import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis.DETERMINISTIC; -import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis.FAILED_DATA_CHANGED; -import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis.NON_DETERMINISTIC; -import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalyzer.Analysis.NOT_RUN; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.DETERMINISTIC; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.FAILED_DATA_CHANGED; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NON_DETERMINISTIC; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NOT_RUN; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; public class TestLimitQueryDeterminismAnalyzer { @@ -95,30 +96,30 @@ public void setup() public void testNotRun() { // Unsupported statement types - assertEquals(analyze("CREATE TABLE test (x varchar, ds varhcar) WITH (partitioned_by = ARRAY[\"ds\"])"), NOT_RUN); - assertEquals(analyze("SELECT * FROM source LIMIT 10"), NOT_RUN); + assertAnalysis("CREATE TABLE test (x varchar, ds varhcar) WITH (partitioned_by = ARRAY[\"ds\"])", NOT_RUN); + assertAnalysis("SELECT * FROM source LIMIT 10", NOT_RUN); // Order by clause - assertEquals(analyze("INSERT INTO test SELECT * FROM source UNION ALL SELECT * FROM source ORDER BY 1 LIMIT 1000"), NOT_RUN); - assertEquals(analyze("INSERT INTO test SELECT * FROM source ORDER BY 1 LIMIT 1000"), NOT_RUN); + assertAnalysis("INSERT INTO test SELECT * FROM source UNION ALL SELECT * FROM source ORDER BY 1 LIMIT 1000", NOT_RUN); + assertAnalysis("INSERT INTO test SELECT * FROM source ORDER BY 1 LIMIT 1000", NOT_RUN); // not outer limit clause - assertEquals(analyze("INSERT INTO test SELECT * FROM source UNION ALL SELECT * FROM source"), NOT_RUN); - assertEquals(analyze("INSERT INTO test SELECT * FROM source"), NOT_RUN); - assertEquals(analyze("INSERT INTO test SELECT * FROM (SELECT * FROM source LIMIT 1000)"), NOT_RUN); + assertAnalysis("INSERT INTO test SELECT * FROM source UNION ALL SELECT * FROM source", NOT_RUN); + assertAnalysis("INSERT INTO test SELECT * FROM source", NOT_RUN); + assertAnalysis("INSERT INTO test SELECT * FROM (SELECT * FROM source LIMIT 1000)", NOT_RUN); } @Test public void testNonDeterministic() { rowCount.set(1001); - assertEquals(analyze("INSERT INTO test SELECT * FROM source LIMIT 1000"), NON_DETERMINISTIC); + assertAnalysis("INSERT INTO test SELECT * FROM source LIMIT 1000", NON_DETERMINISTIC); assertRowCountQuery("SELECT count(1) FROM (SELECT * FROM source)"); - assertEquals(analyze("CREATE TABLE test AS (WITH f AS (select * from g) ((SELECT * FROM source UNION ALL SELECT * FROM source LIMIT 1000)))"), NON_DETERMINISTIC); + assertAnalysis("CREATE TABLE test AS (WITH f AS (select * from g) ((SELECT * FROM source UNION ALL SELECT * FROM source LIMIT 1000)))", NON_DETERMINISTIC); assertRowCountQuery("SELECT count(1) FROM (WITH f AS (select * from g) SELECT * FROM source UNION ALL SELECT * FROM source)"); - assertEquals(analyze("CREATE TABLE test AS (WITH f AS (select * from g) (SELECT * FROM source LIMIT 1000))"), NON_DETERMINISTIC); + assertAnalysis("CREATE TABLE test AS (WITH f AS (select * from g) (SELECT * FROM source LIMIT 1000))", NON_DETERMINISTIC); assertRowCountQuery("SELECT count(1) FROM (WITH f AS (select * from g) SELECT * FROM source)"); } @@ -126,26 +127,36 @@ public void testNonDeterministic() public void testDeterministic() { rowCount.set(1000); - assertEquals(analyze("INSERT INTO test SELECT * FROM source LIMIT 1000"), DETERMINISTIC); + assertAnalysis("INSERT INTO test SELECT * FROM source LIMIT 1000", DETERMINISTIC); } @Test public void testFailedDataChanged() { rowCount.set(999); - assertEquals(analyze("INSERT INTO test SELECT * FROM source LIMIT 1000"), FAILED_DATA_CHANGED); + assertAnalysis("INSERT INTO test SELECT * FROM source LIMIT 1000", FAILED_DATA_CHANGED); } - private Analysis analyze(String query) + private void assertAnalysis(String query, LimitQueryDeterminismAnalysis expectedAnalysis) { - return analyzer.analyze( + VerificationContext verificationContext = new VerificationContext(); + LimitQueryDeterminismAnalysis analysis = analyzer.analyze( new QueryBundle( - Optional.of(TABLE_NAME), + TABLE_NAME, ImmutableList.of(), sqlParser.createStatement(query, PARSING_OPTIONS), ImmutableList.of(), CONTROL), - ROW_COUNT_WITH_LIMIT); + ROW_COUNT_WITH_LIMIT, + verificationContext); + + assertEquals(analysis, expectedAnalysis); + if (expectedAnalysis == NOT_RUN) { + assertFalse(verificationContext.getLimitQueryAnalysisQueryId().isPresent()); + } + else { + assertTrue(verificationContext.getLimitQueryAnalysisQueryId().isPresent()); + } } private void assertRowCountQuery(String expectedQuery) diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java index a27572f8c8eef..bae405c84c73e 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java @@ -40,7 +40,9 @@ public void testDefault() .setQueryRepetitions(1) .setRelativeErrorMargin(1e-4) .setAbsoluteErrorMargin(1e-12) - .setRunTearDownOnResultMismatch(false) + .setRunTeardownOnResultMismatch(false) + .setRunTeardownForDeterminismAnalysis(false) + .setMaxDeterminismAnalysisRuns(2) .setEnableLimitQueryDeterminismAnalyzer(true) .setVerificationResubmissionLimit(2)); } @@ -62,6 +64,8 @@ public void testExplicitPropertyMappings() .put("relative-error-margin", "2e-5") .put("absolute-error-margin", "1e-14") .put("run-teardown-on-result-mismatch", "true") + .put("run-teardown-for-determinism-analysis", "true") + .put("max-determinism-analysis-runs", "3") .put("enable-limit-query-determinism-analyzer", "false") .put("verification-resubmission.limit", "1") .build(); @@ -78,7 +82,9 @@ public void testExplicitPropertyMappings() .setQueryRepetitions(3) .setRelativeErrorMargin(2e-5) .setAbsoluteErrorMargin(1e-14) - .setRunTearDownOnResultMismatch(true) + .setRunTeardownOnResultMismatch(true) + .setRunTeardownForDeterminismAnalysis(true) + .setMaxDeterminismAnalysisRuns(3) .setEnableLimitQueryDeterminismAnalyzer(false) .setVerificationResubmissionLimit(1); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java index 4bc59454efc08..810a2d62a4810 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java @@ -77,7 +77,7 @@ public QueryResult execute(Statement statement, QueryStage queryStage, Re private static final String TABLE_NAME = "test"; private static final int MAX_BUCKETS_PER_WRITER = 100; private static final QueryBundle TEST_BUNDLE = new QueryBundle( - Optional.of(QualifiedName.of(TABLE_NAME)), + QualifiedName.of(TABLE_NAME), ImmutableList.of(), new SqlParser(new SqlParserOptions().allowIdentifierSymbol(AT_SIGN, COLON)).createStatement( "INSERT INTO test SELECT * FROM source",