Verifier: Improve determinism analysis and checksum query recording

presto-docs/src/main/sphinx/installation/verifier.rst

@@ -20,14 +20,15 @@ Create a MySQL database with the following table and load it with the queries yo
         control_catalog varchar(256) NOT NULL,
         control_schema varchar(256) NOT NULL,
         control_query text NOT NULL,
+        control_username varchar(256) DEFAULT NULL,
+        control_password varchar(256) DEFAULT NULL,
+        control_session_properties text DEFAULT NULL,
         test_catalog varchar(256) NOT NULL,
         test_schema varchar(256) NOT NULL,
         test_query text NOT NULL,
-        control_username varchar(256) NOT NULL DEFAULT 'verifier-test',
-        control_password varchar(256) DEFAULT NULL,
-        test_username varchar(256) NOT NULL DEFAULT 'verifier-test',
+        test_username varchar(256) DEFAULT NULL,
         test_password varchar(256) DEFAULT NULL,
-        session_properties_json varchar(2048) DEFAULT NULL)
+        test_session_properties text DEFAULT NULL)
 
 Next, create a properties file to configure the verifier:
 
@@ -50,32 +51,36 @@ make it executable with ``chmod +x``, then run it:
 Configuration Reference
 -----------------------
 
-================================= =======================================================================
-Name                              Description
-================================= =======================================================================
-``control.timeout``               The maximum execution time of the control queries.
-``test.timeout``                  The maximum execution time of the test queries.
-``metadata.timeout``              The maximum execution time of the queries that are required for
-                                  obtaining table metadata or rewriting queries.
-``checksum.timeout``              The maximum execution time of the queries that computes checksum for
-                                  the control and the test results.
-``whitelist``                     A comma-separated list that specifies names of the queries within the
-                                  suite to verify.
-``blacklist``                     A comma-separated list that specifies names of the queries to be
-                                  excluded from suite. ``blacklist`` is applied after ``whitelist``.
-``source-query.table-name``       Specifies the MySQL table from which to read the source queries for
-                                  verification.
-``event-clients``                 A comma-separated list that specifies where the output events should be
-                                  emitted. Valid individual values are ``json`` and ``human-readable``.
-``json.log-file``                 Specifies the output files for JSON events. If ``json`` is specified in
-                                  ``event-clients`` but this property is not set, JSON events are emitted
-                                  to ``stdout``.
-``human-readable.log-file``       Specifies the output files for human readable events. If
-                                  ``human-readable`` is specified in ``event-clients`` but this property
-                                  is not set, human readable events are emitted to ``stdout``.
-``max-concurrency``               Specifies the maximum concurrent verification. Alternatively speaking,
-                                  the maximum concurrent queries that will be submitted to control and
-                                  test clusters combined.
-``relative-error-margin``         Specified the maximum tolerable relative error between control and test
-                                  queries for floating point columns.
-================================= =======================================================================
+=========================================== =======================================================================
+Name                                        Description
+=========================================== =======================================================================
+``control.timeout``                         The maximum execution time of the control queries.
+``test.timeout``                            The maximum execution time of the test queries.
+``metadata.timeout``                        The maximum execution time of the queries that are required for
+                                            obtaining table metadata or rewriting queries.
+``checksum.timeout``                        The maximum execution time of the queries that computes checksum for
+                                            the control and the test results.
+``whitelist``                               A comma-separated list that specifies names of the queries within the
+                                            suite to verify.
+``blacklist``                               A comma-separated list that specifies names of the queries to be
+                                            excluded from suite. ``blacklist`` is applied after ``whitelist``.
+``source-query.table-name``                 Specifies the MySQL table from which to read the source queries for
+                                            verification.
+``event-clients``                           A comma-separated list that specifies where the output events should be
+                                            emitted. Valid individual values are ``json`` and ``human-readable``.
+``json.log-file``                           Specifies the output files for JSON events. If ``json`` is specified in
+                                            ``event-clients`` but this property is not set, JSON events are emitted
+                                            to ``stdout``.
+``human-readable.log-file``                 Specifies the output files for human readable events. If
+                                            ``human-readable`` is specified in ``event-clients`` but this property
+                                            is not set, human readable events are emitted to ``stdout``.
+``max-concurrency``                         Specifies the maximum concurrent verification. Alternatively speaking,
+                                            the maximum concurrent queries that will be submitted to control and
+                                            test clusters combined.
+``relative-error-margin``                   Specified the maximum tolerable relative error between control and test
+                                            queries for floating point columns.
+``max-determinism-analysis-runs``           Maximum number of reruns of the control queries in case of a result
+                                            mismatch to determine whether the query is deterministic.
+``run-teardown-for-determinism-analysis``   Whether temporary tables created in determinism analysis runs are
+                                            teared down.
+=========================================== =======================================================================

presto-verifier/pom.xml

@@ -69,6 +69,11 @@
             <artifactId>presto-thrift-connector</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>

presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisDetails.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.verifier.event;
+
+import com.facebook.airlift.event.client.EventField;
+import com.facebook.airlift.event.client.EventType;
+import com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.google.common.collect.ImmutableList;
+
+import javax.annotation.concurrent.Immutable;
+
+import java.util.List;
+import java.util.Optional;
+
+@Immutable
+@EventType("DeterminismAnalysisDetails")
+public class DeterminismAnalysisDetails
+{
+    private final List<DeterminismAnalysisRun> runs;
+    private final String limitQueryAnalysis;
+    private final String limitQueryAnalysisQueryId;
+
+    @JsonCreator
+    public DeterminismAnalysisDetails(
+            List<DeterminismAnalysisRun> runs,
+            LimitQueryDeterminismAnalysis limitQueryAnalysis,
+            Optional<String> limitQueryAnalysisQueryId)
+    {
+        this.runs = ImmutableList.copyOf(runs);
+        this.limitQueryAnalysis = limitQueryAnalysis.name();
+        this.limitQueryAnalysisQueryId = limitQueryAnalysisQueryId.orElse(null);
+    }
+
+    @EventField
+    public List<DeterminismAnalysisRun> getRuns()
+    {
+        return runs;
+    }
+
+    @EventField
+    public String getLimitQueryAnalysis()
+    {
+        return limitQueryAnalysis;
+    }
+
+    @EventField
+    public String getLimitQueryAnalysisQueryId()
+    {
+        return limitQueryAnalysisQueryId;
+    }
+}

presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisRun.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.verifier.event;
+
+import com.facebook.airlift.event.client.EventField;
+import com.facebook.airlift.event.client.EventType;
+
+import javax.annotation.concurrent.Immutable;
+
+import java.util.Optional;
+
+import static com.google.common.base.Preconditions.checkState;
+import static java.util.Objects.requireNonNull;
+
+@Immutable
+@EventType("DeterminismAnalysisRun")
+public class DeterminismAnalysisRun
+{
+    private final String tableName;
+    private final String queryId;
+    private final String checksumQueryId;
+
+    private DeterminismAnalysisRun(
+            Optional<String> tableName,
+            Optional<String> queryId,
+            Optional<String> checksumQueryId)
+    {
+        this.tableName = tableName.orElse(null);
+        this.queryId = queryId.orElse(null);
+        this.checksumQueryId = checksumQueryId.orElse(null);
+    }
+
+    @EventField
+    public String getTableName()
+    {
+        return tableName;
+    }
+
+    @EventField
+    public String getQueryId()
+    {
+        return queryId;
+    }
+
+    @EventField
+    public String getChecksumQueryId()
+    {
+        return checksumQueryId;
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    public static class Builder
+    {
+        private String tableName;
+        private String queryId;
+        private String checksumQueryId;
+
+        private Builder()
+        {
+        }
+
+        public Builder setTableName(String tableName)
+        {
+            checkState(this.tableName == null, "tableName is already set");
+            this.tableName = requireNonNull(tableName, "tableName is null");
+            return this;
+        }
+
+        public Builder setQueryId(String queryId)
+        {
+            checkState(this.queryId == null, "queryId is already set");
+            this.queryId = requireNonNull(queryId, "queryId is null");
+            return this;
+        }
+
+        public Builder setChecksumQueryId(String checksumQueryId)
+        {
+            checkState(this.checksumQueryId == null, "checksumQueryId is already set");
+            this.checksumQueryId = requireNonNull(checksumQueryId, "checksumQueryId is null");
+            return this;
+        }
+
+        public DeterminismAnalysisRun build()
+        {
+            return new DeterminismAnalysisRun(Optional.ofNullable(tableName), Optional.ofNullable(queryId), Optional.ofNullable(checksumQueryId));
+        }
+    }
+}

presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java

@@ -48,6 +48,7 @@ public enum EventStatus
 
     private final Boolean deterministic;
     private final String determinismAnalysis;
+    private final DeterminismAnalysisDetails determinismAnalysisDetails;
     private final String resolveMessage;
 
     private final QueryInfo controlQueryInfo;
@@ -66,6 +67,7 @@ public VerifierQueryEvent(
             EventStatus status,
             Optional<SkippedReason> skippedReason,
             Optional<DeterminismAnalysis> determinismAnalysis,
+            Optional<DeterminismAnalysisDetails> determinismAnalysisDetails,
             Optional<String> resolveMessage,
             Optional<QueryInfo> controlQueryInfo,
             Optional<QueryInfo> testQueryInfo,
@@ -81,6 +83,7 @@ public VerifierQueryEvent(
         this.skippedReason = skippedReason.map(SkippedReason::name).orElse(null);
         this.deterministic = determinismAnalysis.filter(d -> !d.isUnknown()).map(DeterminismAnalysis::isDeterministic).orElse(null);
         this.determinismAnalysis = determinismAnalysis.map(DeterminismAnalysis::name).orElse(null);
+        this.determinismAnalysisDetails = determinismAnalysisDetails.orElse(null);
         this.resolveMessage = resolveMessage.orElse(null);
         this.controlQueryInfo = controlQueryInfo.orElse(null);
         this.testQueryInfo = testQueryInfo.orElse(null);
@@ -109,6 +112,7 @@ public static VerifierQueryEvent skipped(
                 Optional.empty(),
                 Optional.empty(),
                 Optional.empty(),
+                Optional.empty(),
                 ImmutableList.of());
     }
 
@@ -155,6 +159,12 @@ public String getDeterminismAnalysis()
         return determinismAnalysis;
     }
 
+    @EventField
+    public DeterminismAnalysisDetails getDeterminismAnalysisDetails()
+    {
+        return determinismAnalysisDetails;
+    }
+
     @EventField
     public String getResolveMessage()
     {