diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ArrayColumnValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ArrayColumnValidator.java index 78071e72f9c39..4cba18d0aa59f 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ArrayColumnValidator.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ArrayColumnValidator.java @@ -53,25 +53,25 @@ public List generateChecksumColumns(Column column) Expression checksum; if (elementType.isOrderable()) { - FunctionCall arraySort = new FunctionCall(QualifiedName.of("array_sort"), ImmutableList.of(column.getIdentifier())); + FunctionCall arraySort = new FunctionCall(QualifiedName.of("array_sort"), ImmutableList.of(column.getExpression())); if (elementType instanceof ArrayType || elementType instanceof RowType) { checksum = new CoalesceExpression( new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(new TryExpression(arraySort))), - new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(column.getIdentifier()))); + new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(column.getExpression()))); } else { checksum = new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(arraySort)); } } else { - checksum = new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(column.getIdentifier())); + checksum = new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(column.getExpression())); } Expression arrayCardinalitySum = new CoalesceExpression( new FunctionCall( QualifiedName.of("sum"), - ImmutableList.of(new FunctionCall(QualifiedName.of("cardinality"), ImmutableList.of(column.getIdentifier())))), + ImmutableList.of(new FunctionCall(QualifiedName.of("cardinality"), ImmutableList.of(column.getExpression())))), new LongLiteral("0")); return ImmutableList.of( @@ -80,7 +80,7 @@ public List generateChecksumColumns(Column column) } @Override - public ColumnMatchResult validate(Column column, ChecksumResult controlResult, ChecksumResult testResult) + public List validate(Column column, ChecksumResult controlResult, ChecksumResult testResult) { String checksumColumnAlias = getChecksumColumnAlias(column); Object controlChecksum = controlResult.getChecksum(checksumColumnAlias); @@ -90,14 +90,15 @@ public ColumnMatchResult validate(Column column, ChecksumResult controlResult, C Object controlCardinalitySum = controlResult.getChecksum(cardinalitySumColumnAlias); Object testCardinalitySum = testResult.getChecksum(cardinalitySumColumnAlias); - return new ColumnMatchResult( + return ImmutableList.of(new ColumnMatchResult( Objects.equals(controlChecksum, testChecksum) && Objects.equals(controlCardinalitySum, testCardinalitySum), + column, format( "control(checksum: %s, cardinality_sum: %s) test(checksum: %s, cardinality_sum: %s)", controlChecksum, controlCardinalitySum, testChecksum, - testCardinalitySum)); + testCardinalitySum))); } private static String getChecksumColumnAlias(Column column) diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java index 00c343cdd8f06..89bedfdc9151a 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java @@ -23,38 +23,26 @@ import com.facebook.presto.verifier.framework.Column; import com.facebook.presto.verifier.framework.Column.Category; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import javax.inject.Inject; +import javax.inject.Provider; import java.util.List; import java.util.Map; import java.util.Map.Entry; import static com.facebook.presto.sql.QueryUtil.simpleQuery; -import static com.facebook.presto.verifier.framework.Column.Category.ARRAY; -import static com.facebook.presto.verifier.framework.Column.Category.FLOATING_POINT; -import static com.facebook.presto.verifier.framework.Column.Category.ROW; -import static com.facebook.presto.verifier.framework.Column.Category.SIMPLE; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.function.Function.identity; public class ChecksumValidator { - private final Map columnValidators; + private final Map> columnValidators; @Inject - public ChecksumValidator( - SimpleColumnValidator simpleColumnValidator, - FloatingPointColumnValidator floatingPointColumnValidator, - ArrayColumnValidator arrayColumnValidator, - RowColumnValidator rowColumnValidator) + public ChecksumValidator(Map> columnValidators) { - this.columnValidators = ImmutableMap.of( - SIMPLE, simpleColumnValidator, - FLOATING_POINT, floatingPointColumnValidator, - ARRAY, arrayColumnValidator, - ROW, rowColumnValidator); + this.columnValidators = columnValidators; } public Query generateChecksumQuery(QualifiedName tableName, List columns) @@ -62,7 +50,7 @@ public Query generateChecksumQuery(QualifiedName tableName, List columns ImmutableList.Builder selectItems = ImmutableList.builder(); selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of()))); for (Column column : columns) { - selectItems.addAll(columnValidators.get(column.getCategory()).generateChecksumColumns(column)); + selectItems.addAll(columnValidators.get(column.getCategory()).get().generateChecksumColumns(column)); } return simpleQuery(new Select(false, selectItems.build()), new Table(tableName)); } @@ -70,7 +58,8 @@ public Query generateChecksumQuery(QualifiedName tableName, List columns public Map getMismatchedColumns(List columns, ChecksumResult controlChecksum, ChecksumResult testChecksum) { return columns.stream() - .collect(toImmutableMap(identity(), column -> columnValidators.get(column.getCategory()).validate(column, controlChecksum, testChecksum))) + .flatMap(column -> columnValidators.get(column.getCategory()).get().validate(column, controlChecksum, testChecksum).stream()) + .collect(toImmutableMap(ColumnMatchResult::getColumn, identity())) .entrySet() .stream() .filter(entry -> !entry.getValue().isMatched()) diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ColumnMatchResult.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ColumnMatchResult.java index d36f97cfb69a1..634678acd58e7 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ColumnMatchResult.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ColumnMatchResult.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.verifier.checksum; +import com.facebook.presto.verifier.framework.Column; + import java.util.Objects; import static com.google.common.base.MoreObjects.toStringHelper; @@ -21,11 +23,13 @@ public class ColumnMatchResult { private final boolean matched; + private final Column column; private final String message; - public ColumnMatchResult(boolean matched, String message) + public ColumnMatchResult(boolean matched, Column column, String message) { this.matched = matched; + this.column = requireNonNull(column, "column is null"); this.message = requireNonNull(message, "message is null"); } @@ -34,6 +38,11 @@ public boolean isMatched() return matched; } + public Column getColumn() + { + return column; + } + public String getMessage() { return message; @@ -50,13 +59,14 @@ public boolean equals(Object obj) } ColumnMatchResult o = (ColumnMatchResult) obj; return Objects.equals(matched, o.matched) && + Objects.equals(column, o.column) && Objects.equals(message, o.message); } @Override public int hashCode() { - return Objects.hash(matched, message); + return Objects.hash(matched, column, message); } @Override @@ -64,6 +74,7 @@ public String toString() { return toStringHelper(this) .add("matched", matched) + .add("column", column) .add("message", message) .toString(); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ColumnValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ColumnValidator.java index 9eb2a33ae23cf..c19712dc82946 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ColumnValidator.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ColumnValidator.java @@ -18,9 +18,9 @@ import java.util.List; -interface ColumnValidator +public interface ColumnValidator { List generateChecksumColumns(Column column); - ColumnMatchResult validate(Column column, ChecksumResult controlResult, ChecksumResult testResult); + List validate(Column column, ChecksumResult controlResult, ChecksumResult testResult); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/FloatingPointColumnValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/FloatingPointColumnValidator.java index 692ce9edfcd9e..bc6fa29322421 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/FloatingPointColumnValidator.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/FloatingPointColumnValidator.java @@ -57,7 +57,7 @@ public FloatingPointColumnValidator(VerifierConfig config) @Override public List generateChecksumColumns(Column column) { - Expression doubleColumn = column.getType().equals(DOUBLE) ? column.getIdentifier() : new Cast(column.getIdentifier(), DOUBLE.getDisplayName()); + Expression doubleColumn = column.getType().equals(DOUBLE) ? column.getExpression() : new Cast(column.getExpression(), DOUBLE.getDisplayName()); Expression positiveInfinity = new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of()); Expression negativeInfinity = new ArithmeticUnaryExpression(MINUS, positiveInfinity); @@ -66,7 +66,7 @@ public List generateChecksumColumns(Column column) new FunctionCall( QualifiedName.of("sum"), Optional.empty(), - Optional.of(new FunctionCall(QualifiedName.of("is_finite"), ImmutableList.of(column.getIdentifier()))), + Optional.of(new FunctionCall(QualifiedName.of("is_finite"), ImmutableList.of(column.getExpression()))), Optional.empty(), false, ImmutableList.of(doubleColumn)), @@ -75,33 +75,33 @@ public List generateChecksumColumns(Column column) new FunctionCall( QualifiedName.of("count"), Optional.empty(), - Optional.of(new FunctionCall(QualifiedName.of("is_nan"), ImmutableList.of(column.getIdentifier()))), + Optional.of(new FunctionCall(QualifiedName.of("is_nan"), ImmutableList.of(column.getExpression()))), Optional.empty(), false, - ImmutableList.of(column.getIdentifier())), + ImmutableList.of(column.getExpression())), Optional.of(delimitedIdentifier(getNanCountColumnAlias(column)))), new SingleColumn( new FunctionCall( QualifiedName.of("count"), Optional.empty(), - Optional.of(new ComparisonExpression(EQUAL, column.getIdentifier(), positiveInfinity)), + Optional.of(new ComparisonExpression(EQUAL, column.getExpression(), positiveInfinity)), Optional.empty(), false, - ImmutableList.of(column.getIdentifier())), + ImmutableList.of(column.getExpression())), Optional.of(delimitedIdentifier(getPositiveInfinityCountColumnAlias(column)))), new SingleColumn( new FunctionCall( QualifiedName.of("count"), Optional.empty(), - Optional.of(new ComparisonExpression(EQUAL, column.getIdentifier(), negativeInfinity)), + Optional.of(new ComparisonExpression(EQUAL, column.getExpression(), negativeInfinity)), Optional.empty(), false, - ImmutableList.of(column.getIdentifier())), + ImmutableList.of(column.getExpression())), Optional.of(delimitedIdentifier(getNegativeInfinityCountColumnAlias(column))))); } @Override - public ColumnMatchResult validate(Column column, ChecksumResult controlResult, ChecksumResult testResult) + public List validate(Column column, ChecksumResult controlResult, ChecksumResult testResult) { checkArgument( controlResult.getRowCount() == testResult.getRowCount(), @@ -123,8 +123,9 @@ public ColumnMatchResult validate(Column column, ChecksumResult controlResult, C if (!Objects.equals(controlNanCount, testNanCount) || !Objects.equals(controlPositiveInfinityCount, testPositiveInfinityCount) || !Objects.equals(controlNegativeInfinityCount, testNegativeInfinityCount)) { - return new ColumnMatchResult( + return ImmutableList.of(new ColumnMatchResult( false, + column, format( "control(NaN: %s, +infinity: %s, -infinity: %s) test(NaN: %s, +infinity: %s, -infinity: %s)", controlNanCount, @@ -132,15 +133,16 @@ public ColumnMatchResult validate(Column column, ChecksumResult controlResult, C controlNegativeInfinityCount, testNanCount, testPositiveInfinityCount, - testNegativeInfinityCount)); + testNegativeInfinityCount))); } Object controlSumObject = controlResult.getChecksum(sumColumnAlias); Object testSumObject = testResult.getChecksum(sumColumnAlias); if (controlSumObject == null || testSumObject == null) { - return new ColumnMatchResult( + return ImmutableList.of(new ColumnMatchResult( controlSumObject == null && testSumObject == null, - format("control(sum: %s) test(sum: %s)", controlSumObject, testSumObject)); + column, + format("control(sum: %s) test(sum: %s)", controlSumObject, testSumObject))); } // Implementation according to http://floating-point-gui.de/errors/comparison/ @@ -149,9 +151,10 @@ public ColumnMatchResult validate(Column column, ChecksumResult controlResult, C // Fail if either sum is NaN or Infinity if (isNaN(controlSum) || isNaN(testSum) || isInfinite(controlSum) || isInfinite(testSum)) { - return new ColumnMatchResult( + return ImmutableList.of(new ColumnMatchResult( false, - format("control(sum: %s) test(sum: %s)", controlSum, testSum)); + column, + format("control(sum: %s) test(sum: %s)", controlSum, testSum))); } // Use absolute error margin if either control sum or test sum is 0 @@ -159,17 +162,19 @@ public ColumnMatchResult validate(Column column, ChecksumResult controlResult, C double controlMean = controlSum / controlResult.getRowCount(); double testMean = testSum / controlResult.getRowCount(); double difference = abs(controlMean - testMean); - return new ColumnMatchResult( + return ImmutableList.of(new ColumnMatchResult( difference < absoluteErrorMargin, - format("control(mean: %s) test(mean: %s) difference: %s", controlMean, testMean, difference)); + column, + format("control(mean: %s) test(mean: %s) difference: %s", controlMean, testMean, difference))); } // Use relative error margin for the common cases double difference = abs(controlSum - testSum); double relativeError = difference / min((abs(controlSum) + abs(testSum)) / 2, Double.MAX_VALUE); - return new ColumnMatchResult( + return ImmutableList.of(new ColumnMatchResult( relativeError < relativeErrorMargin, - format("control(sum: %s) test(sum: %s) relative error: %s", controlSum, testSum, relativeError)); + column, + format("control(sum: %s) test(sum: %s) relative error: %s", controlSum, testSum, relativeError))); } private static String getSumColumnAlias(Column column) diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/RowColumnValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/RowColumnValidator.java index be698837460ad..34186316c245a 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/RowColumnValidator.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/RowColumnValidator.java @@ -17,34 +17,30 @@ import com.facebook.presto.spi.type.RowType.Field; import com.facebook.presto.sql.tree.DereferenceExpression; import com.facebook.presto.sql.tree.Expression; -import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.LongLiteral; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.SingleColumn; import com.facebook.presto.sql.tree.SubscriptExpression; import com.facebook.presto.verifier.framework.Column; import com.google.common.collect.ImmutableList; import javax.inject.Inject; +import javax.inject.Provider; import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.IntStream; +import java.util.Map; -import static com.facebook.presto.verifier.framework.VerifierUtil.delimitedIdentifier; import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.util.stream.Collectors.joining; public class RowColumnValidator implements ColumnValidator { + private final Map> columnValidators; + @Inject - public RowColumnValidator() + public RowColumnValidator(Map> columnValidators) { + this.columnValidators = columnValidators; } @Override @@ -52,48 +48,33 @@ public List generateChecksumColumns(Column column) { checkColumnType(column); - Expression checksum = new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(column.getIdentifier())); - ImmutableList.Builder columnsBuilder = ImmutableList.builder(); - columnsBuilder.add(new SingleColumn(checksum, Optional.of(delimitedIdentifier(getChecksumAlias(column))))); List fields = getFields(column); - for (int i = 0; i < getFields(column).size(); i++) { + for (int i = 0; i < fields.size(); i++) { Field field = fields.get(i); - - Expression fieldExpression; - - if (field.getName().isPresent()) { - fieldExpression = new DereferenceExpression(column.getIdentifier(), new Identifier(field.getName().get())); - } - else { - fieldExpression = new SubscriptExpression(column.getIdentifier(), new LongLiteral(String.valueOf(i + 1))); - } - - Expression fieldChecksum = new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(fieldExpression)); - columnsBuilder.add(new SingleColumn(fieldChecksum, Optional.of(delimitedIdentifier(getChecksumAlias(column, field, i))))); + Column fieldColumn = getFieldAsColumn(column, field, i); + List fieldColumns = columnValidators.get(fieldColumn.getCategory()).get().generateChecksumColumns(fieldColumn); + columnsBuilder.addAll(fieldColumns); } return columnsBuilder.build(); } @Override - public ColumnMatchResult validate(Column column, ChecksumResult controlResult, ChecksumResult testResult) + public List validate(Column column, ChecksumResult controlResult, ChecksumResult testResult) { checkColumnType(column); + ImmutableList.Builder resultsBuilder = ImmutableList.builder(); List fields = getFields(column); - List aliases = ImmutableList.builder() - .add(getChecksumAlias(column)) - .addAll(IntStream.range(0, fields.size()) - .mapToObj(index -> getChecksumAlias(column, fields.get(index), index)) - .collect(Collectors.toList())) - .build(); - - return new ColumnMatchResult( - aliases.stream().allMatch(alias -> Objects.equals(controlResult.getChecksum(alias), testResult.getChecksum(alias))), - prepareMatchMessage(aliases, controlResult, testResult)); + for (int i = 0; i < fields.size(); i++) { + Column fieldColumn = getFieldAsColumn(column, fields.get(i), i); + resultsBuilder.addAll(columnValidators.get(fieldColumn.getCategory()).get().validate(fieldColumn, controlResult, testResult)); + } + + return resultsBuilder.build(); } private static void checkColumnType(Column column) @@ -106,30 +87,12 @@ private static List getFields(Column column) return ((RowType) column.getType()).getFields(); } - private static String getChecksumAlias(Column column) + private static Column getFieldAsColumn(Column column, Field field, int fieldIndex) { - return column.getName() + "_checksum"; - } - - private static String getChecksumAlias(Column column, Field field, int fieldIndex) - { - return column.getName() + "$" + field.getName().orElse("$col" + (fieldIndex + 1)) + "_checksum"; - } - - private static String prepareMatchMessage(List aliases, ChecksumResult controlResult, ChecksumResult testResult) - { - aliases = aliases.stream() - .filter(alias -> !Objects.equals(controlResult.getChecksum(alias), testResult.getChecksum(alias))) - .collect(Collectors.toList()); - - String control = aliases.stream() - .map(alias -> alias + ": " + controlResult.getChecksum(alias)) - .collect(joining(", ")); - - String test = aliases.stream() - .map(alias -> alias + ": " + testResult.getChecksum(alias)) - .collect(joining(", ")); + Expression fieldExpression = field.getName() + .map(name -> new DereferenceExpression(column.getExpression(), new Identifier(field.getName().get()))) + .orElseGet(() -> new SubscriptExpression(column.getExpression(), new LongLiteral(String.valueOf(fieldIndex + 1)))); - return format("control(%s) test(%s)", control, test); + return Column.create(column.getName() + "$" + field.getName().orElse("$col" + (fieldIndex + 1)), fieldExpression, field.getType()); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/SimpleColumnValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/SimpleColumnValidator.java index 3fa3d0e01d4e2..85d588ed6dfa5 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/SimpleColumnValidator.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/SimpleColumnValidator.java @@ -41,19 +41,20 @@ public List generateChecksumColumns(Column column) { return ImmutableList.of( new SingleColumn( - new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(column.getIdentifier())), + new FunctionCall(QualifiedName.of("checksum"), ImmutableList.of(column.getExpression())), Optional.of(delimitedIdentifier(getChecksumColumnAlias(column))))); } @Override - public ColumnMatchResult validate(Column column, ChecksumResult controlResult, ChecksumResult testResult) + public List validate(Column column, ChecksumResult controlResult, ChecksumResult testResult) { String checksumColumnAlias = getChecksumColumnAlias(column); Object controlChecksum = controlResult.getChecksum(checksumColumnAlias); Object testChecksum = testResult.getChecksum(checksumColumnAlias); - return new ColumnMatchResult( + return ImmutableList.of(new ColumnMatchResult( Objects.equals(controlChecksum, testChecksum), - format("control(checksum: %s) test(checksum: %s)", controlChecksum, testChecksum)); + column, + format("control(checksum: %s) test(checksum: %s)", controlChecksum, testChecksum))); } private static String getChecksumColumnAlias(Column column) diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java index c45180a2220c3..7b3a25aa7ce94 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java @@ -17,8 +17,7 @@ import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; -import com.facebook.presto.sql.tree.Identifier; -import com.google.common.annotations.VisibleForTesting; +import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableSet; import java.sql.ResultSet; @@ -49,13 +48,14 @@ public enum Category private static final Set FLOATING_POINT_TYPES = ImmutableSet.of(DOUBLE, REAL); private final String name; + private final Expression expression; private final Category category; private final Type type; - @VisibleForTesting - public Column(String name, Category category, Type type) + private Column(String name, Expression expression, Category category, Type type) { this.name = requireNonNull(name, "name is null"); + this.expression = requireNonNull(expression, "expression is null"); this.category = requireNonNull(category, "kind is null"); this.type = requireNonNull(type, "type is null"); } @@ -65,9 +65,9 @@ public String getName() return name; } - public Identifier getIdentifier() + public Expression getExpression() { - return delimitedIdentifier(name); + return expression; } public Category getCategory() @@ -83,7 +83,13 @@ public Type getType() public static Column fromResultSet(TypeManager typeManager, ResultSet resultSet) throws SQLException { + String columnName = resultSet.getString("Column"); Type type = typeManager.getType(parseTypeSignature(resultSet.getString("Type"))); + return create(columnName, delimitedIdentifier(columnName), type); + } + + public static Column create(String name, Expression expression, Type type) + { Category category; if (FLOATING_POINT_TYPES.contains(type)) { category = FLOATING_POINT; @@ -97,7 +103,7 @@ else if (type instanceof RowType) { else { category = SIMPLE; } - return new Column(resultSet.getString("Column"), category, type); + return new Column(name, expression, category, type); } @Override diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java index 5425b840ffab5..88c03c3cd8666 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java @@ -35,9 +35,11 @@ import com.facebook.presto.verifier.annotation.ForTest; import com.facebook.presto.verifier.checksum.ArrayColumnValidator; import com.facebook.presto.verifier.checksum.ChecksumValidator; +import com.facebook.presto.verifier.checksum.ColumnValidator; import com.facebook.presto.verifier.checksum.FloatingPointColumnValidator; import com.facebook.presto.verifier.checksum.RowColumnValidator; import com.facebook.presto.verifier.checksum.SimpleColumnValidator; +import com.facebook.presto.verifier.framework.Column.Category; import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; import com.facebook.presto.verifier.prestoaction.VerificationPrestoActionModule; import com.facebook.presto.verifier.resolver.FailureResolverFactory; @@ -54,6 +56,7 @@ import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.MapBinder; import javax.inject.Provider; import javax.inject.Singleton; @@ -65,6 +68,10 @@ import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.presto.verifier.framework.Column.Category.ARRAY; +import static com.facebook.presto.verifier.framework.Column.Category.FLOATING_POINT; +import static com.facebook.presto.verifier.framework.Column.Category.ROW; +import static com.facebook.presto.verifier.framework.Column.Category.SIMPLE; import static com.google.inject.Scopes.SINGLETON; import static com.google.inject.multibindings.Multibinder.newSetBinder; import static java.util.Objects.requireNonNull; @@ -141,10 +148,11 @@ protected final void setup(Binder binder) binder.bind(VerificationManager.class).in(SINGLETON); binder.bind(VerificationFactory.class).in(SINGLETON); binder.bind(ChecksumValidator.class).in(SINGLETON); - binder.bind(SimpleColumnValidator.class).in(SINGLETON); - binder.bind(FloatingPointColumnValidator.class).in(SINGLETON); - binder.bind(ArrayColumnValidator.class).in(SINGLETON); - binder.bind(RowColumnValidator.class).in(SINGLETON); + MapBinder columnValidatorBinder = MapBinder.newMapBinder(binder, Category.class, ColumnValidator.class); + columnValidatorBinder.addBinding(SIMPLE).to(SimpleColumnValidator.class).in(SINGLETON); + columnValidatorBinder.addBinding(FLOATING_POINT).to(FloatingPointColumnValidator.class).in(SINGLETON); + columnValidatorBinder.addBinding(ARRAY).to(ArrayColumnValidator.class).in(SINGLETON); + columnValidatorBinder.addBinding(ROW).to(RowColumnValidator.class).in(SINGLETON); binder.bind(new TypeLiteral>>() {}).toProvider(new CustomQueryFilterProvider(customQueryFilterClasses)); binder.bind(new TypeLiteral>() {}).toInstance(tablePropertyOverrides); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java index 3381bebda8661..ecb4164266180 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java @@ -18,14 +18,28 @@ import com.facebook.presto.testing.mysql.MySqlOptions; import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.StandaloneQueryRunner; +import com.facebook.presto.verifier.checksum.ArrayColumnValidator; +import com.facebook.presto.verifier.checksum.ChecksumValidator; +import com.facebook.presto.verifier.checksum.ColumnValidator; +import com.facebook.presto.verifier.checksum.FloatingPointColumnValidator; +import com.facebook.presto.verifier.checksum.RowColumnValidator; +import com.facebook.presto.verifier.checksum.SimpleColumnValidator; +import com.facebook.presto.verifier.framework.Column; +import com.facebook.presto.verifier.framework.VerifierConfig; import com.facebook.presto.verifier.source.MySqlSourceQueryConfig; import com.facebook.presto.verifier.source.VerifierDao; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.airlift.units.Duration; import org.jdbi.v3.core.Handle; import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.sqlobject.SqlObjectPlugin; +import javax.inject.Provider; + +import java.util.HashMap; +import java.util.Map; + import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static java.util.concurrent.TimeUnit.SECONDS; @@ -96,4 +110,16 @@ public static void truncateVerifierQueries(Handle handle) { handle.execute("DELETE FROM verifier_queries"); } + + public static ChecksumValidator createChecksumValidator(VerifierConfig verifierConfig) + { + Map> lazyValidators = new HashMap<>(); + Map> validators = ImmutableMap.of( + Column.Category.SIMPLE, SimpleColumnValidator::new, + Column.Category.FLOATING_POINT, () -> new FloatingPointColumnValidator(verifierConfig), + Column.Category.ARRAY, ArrayColumnValidator::new, + Column.Category.ROW, () -> new RowColumnValidator(lazyValidators)); + lazyValidators.putAll(validators); + return new ChecksumValidator(validators); + } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java index b992aecb7f50c..9fb8a13646d30 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java @@ -17,9 +17,12 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.SqlVarbinary; +import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; +import com.facebook.presto.sql.tree.DereferenceExpression; +import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Query; import com.facebook.presto.sql.tree.Statement; @@ -44,11 +47,9 @@ import static com.facebook.presto.sql.SqlFormatter.formatSql; import static com.facebook.presto.sql.parser.IdentifierSymbol.AT_SIGN; import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; -import static com.facebook.presto.verifier.framework.Column.Category.ARRAY; -import static com.facebook.presto.verifier.framework.Column.Category.FLOATING_POINT; -import static com.facebook.presto.verifier.framework.Column.Category.ROW; -import static com.facebook.presto.verifier.framework.Column.Category.SIMPLE; +import static com.facebook.presto.verifier.VerifierTestUtil.createChecksumValidator; import static com.facebook.presto.verifier.framework.VerifierUtil.PARSING_OPTIONS; +import static com.facebook.presto.verifier.framework.VerifierUtil.delimitedIdentifier; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -60,14 +61,14 @@ public class TestChecksumValidator new FunctionManager(typeRegistry, new BlockEncodingManager(typeRegistry), new FeaturesConfig()); } - private static final Column BIGINT_COLUMN = new Column("bigint", SIMPLE, BIGINT); - private static final Column VARCHAR_COLUMN = new Column("varchar", SIMPLE, VARCHAR); - private static final Column DOUBLE_COLUMN = new Column("double", FLOATING_POINT, DOUBLE); - private static final Column REAL_COLUMN = new Column("real", FLOATING_POINT, REAL); - private static final Column INT_ARRAY_COLUMN = new Column("int_array", ARRAY, new ArrayType(INTEGER)); - private static final Column ROW_ARRAY_COLUMN = new Column("row_array", ARRAY, typeRegistry.getType(parseTypeSignature("array(row(a int,b varchar))"))); - private static final Column MAP_ARRAY_COLUMN = new Column("map_array", ARRAY, typeRegistry.getType(parseTypeSignature("array(map(int,varchar))"))); - private static final Column ROW_COLUMN = new Column("row", ROW, typeRegistry.getType(parseTypeSignature("row(a int,double,array(row(a int,b varchar)),b varchar)"))); + private static final Column BIGINT_COLUMN = createColumn("bigint", BIGINT); + private static final Column VARCHAR_COLUMN = createColumn("varchar", VARCHAR); + private static final Column DOUBLE_COLUMN = createColumn("double", DOUBLE); + private static final Column REAL_COLUMN = createColumn("real", REAL); + private static final Column INT_ARRAY_COLUMN = createColumn("int_array", new ArrayType(INTEGER)); + private static final Column ROW_ARRAY_COLUMN = createColumn("row_array", typeRegistry.getType(parseTypeSignature("array(row(a int,b varchar))"))); + private static final Column MAP_ARRAY_COLUMN = createColumn("map_array", typeRegistry.getType(parseTypeSignature("array(map(int,varchar))"))); + private static final Column ROW_COLUMN = createColumn("row", typeRegistry.getType(parseTypeSignature("row(i int, varchar, d double, a array(int), r row(double, b bigint))"))); private static final double RELATIVE_ERROR_MARGIN = 1e-4; private static final double ABSOLUTE_ERROR_MARGIN = 1e-12; @@ -79,13 +80,26 @@ public class TestChecksumValidator .put("real_pos_inf_count", 3L) .put("real_neg_inf_count", 4L) .build(); + private static final Map ROW_COLUMN_CHECKSUMS = ImmutableMap.builder() + .put("row$i_checksum", new SqlVarbinary(new byte[] {0xa})) + .put("row$$col2_checksum", new SqlVarbinary(new byte[] {0xb})) + .put("row$d_nan_count", 2L) + .put("row$d_pos_inf_count", 3L) + .put("row$d_neg_inf_count", 4L) + .put("row$d_sum", 0.0) + .put("row$a_checksum", new SqlVarbinary(new byte[] {0xc})) + .put("row$a_cardinality_sum", 2L) + .put("row$r$$col1_nan_count", 2L) + .put("row$r$$col1_pos_inf_count", 3L) + .put("row$r$$col1_neg_inf_count", 4L) + .put("row$r$$col1_sum", 0.0) + .put("row$r$b_checksum", new SqlVarbinary(new byte[] {0xd})) + .build(); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); - private final ChecksumValidator checksumValidator = new ChecksumValidator( - new SimpleColumnValidator(), - new FloatingPointColumnValidator(new VerifierConfig().setRelativeErrorMargin(RELATIVE_ERROR_MARGIN).setAbsoluteErrorMargin(ABSOLUTE_ERROR_MARGIN)), - new ArrayColumnValidator(), - new RowColumnValidator()); + private final ChecksumValidator checksumValidator = createChecksumValidator(new VerifierConfig() + .setRelativeErrorMargin(RELATIVE_ERROR_MARGIN) + .setAbsoluteErrorMargin(ABSOLUTE_ERROR_MARGIN)); @Test public void testChecksumQuery() @@ -120,11 +134,19 @@ public void testChecksumQuery() ", COALESCE(\"sum\"(\"cardinality\"(\"row_array\")), 0) \"row_array_cardinality_sum\"" + ", \"checksum\"(\"map_array\") \"map_array_checksum\"\n" + ", COALESCE(\"sum\"(\"cardinality\"(\"map_array\")), 0) \"map_array_cardinality_sum\"" + - ", \"checksum\"(\"row\") \"row_checksum\"\n " + - ", \"checksum\"(\"row\".a) \"row$a_checksum\"\n " + + ", \"checksum\"(\"row\".i) \"row$i_checksum\"\n " + ", \"checksum\"(\"row\"[2]) \"row$$col2_checksum\"\n " + - ", \"checksum\"(\"row\"[3]) \"row$$col3_checksum\"\n " + - ", \"checksum\"(\"row\".b) \"row$b_checksum\"\n " + + ", \"sum\"(\"row\".d) FILTER (WHERE \"is_finite\"(\"row\".d)) \"row$d_sum\"" + + ", \"count\"(\"row\".d) FILTER (WHERE \"is_nan\"(\"row\".d)) \"row$d_nan_count\"" + + ", \"count\"(\"row\".d) FILTER (WHERE (\"row\".d = \"infinity\"())) \"row$d_pos_inf_count\"" + + ", \"count\"(\"row\".d) FILTER (WHERE (\"row\".d = -\"infinity\"())) \"row$d_neg_inf_count\"" + + ", \"checksum\"(\"array_sort\"(\"row\".a)) \"row$a_checksum\"" + + ", COALESCE(\"sum\"(\"cardinality\"(\"row\".a)), 0) \"row$a_cardinality_sum\"" + + ", \"sum\"(\"row\".r[1]) FILTER (WHERE \"is_finite\"(\"row\".r[1])) \"row$r$$col1_sum\"" + + ", \"count\"(\"row\".r[1]) FILTER (WHERE \"is_nan\"(\"row\".r[1])) \"row$r$$col1_nan_count\"" + + ", \"count\"(\"row\".r[1]) FILTER (WHERE (\"row\".r[1] = \"infinity\"())) \"row$r$$col1_pos_inf_count\"" + + ", \"count\"(\"row\".r[1]) FILTER (WHERE (\"row\".r[1] = -\"infinity\"())) \"row$r$$col1_neg_inf_count\"" + + ", \"checksum\"(\"row\".r.b) \"row$r$b_checksum\"" + "FROM\n" + " test:di", PARSING_OPTIONS); @@ -155,8 +177,8 @@ public void testSimple() assertEquals( checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), ImmutableMap.builder() - .put(BIGINT_COLUMN, new ColumnMatchResult(false, "control(checksum: 0a) test(checksum: 1a)")) - .put(VARCHAR_COLUMN, new ColumnMatchResult(false, "control(checksum: 0b) test(checksum: 1b)")) + .put(BIGINT_COLUMN, new ColumnMatchResult(false, BIGINT_COLUMN, "control(checksum: 0a) test(checksum: 1a)")) + .put(VARCHAR_COLUMN, new ColumnMatchResult(false, VARCHAR_COLUMN, "control(checksum: 0b) test(checksum: 1b)")) .build()); } @@ -198,8 +220,8 @@ public void testFloatingPoint() assertEquals( checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), ImmutableMap.builder() - .put(DOUBLE_COLUMN, new ColumnMatchResult(false, "control(NaN: 2, +infinity: 3, -infinity: 4) test(NaN: 0, +infinity: 3, -infinity: 4)")) - .put(REAL_COLUMN, new ColumnMatchResult(false, "control(NaN: 2, +infinity: 3, -infinity: 4) test(NaN: 2, +infinity: 0, -infinity: 4)")) + .put(DOUBLE_COLUMN, new ColumnMatchResult(false, DOUBLE_COLUMN, "control(NaN: 2, +infinity: 3, -infinity: 4) test(NaN: 0, +infinity: 3, -infinity: 4)")) + .put(REAL_COLUMN, new ColumnMatchResult(false, REAL_COLUMN, "control(NaN: 2, +infinity: 3, -infinity: 4) test(NaN: 2, +infinity: 0, -infinity: 4)")) .build()); testChecksum = new ChecksumResult( @@ -217,8 +239,8 @@ public void testFloatingPoint() assertEquals( checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), ImmutableMap.builder() - .put(DOUBLE_COLUMN, new ColumnMatchResult(false, "control(NaN: 2, +infinity: 3, -infinity: 4) test(NaN: 2, +infinity: 3, -infinity: 0)")) - .put(REAL_COLUMN, new ColumnMatchResult(false, "control(sum: 1.0) test(sum: 0.9999) relative error: 1.0000500025000149E-4")) + .put(DOUBLE_COLUMN, new ColumnMatchResult(false, DOUBLE_COLUMN, "control(NaN: 2, +infinity: 3, -infinity: 4) test(NaN: 2, +infinity: 3, -infinity: 0)")) + .put(REAL_COLUMN, new ColumnMatchResult(false, REAL_COLUMN, "control(sum: 1.0) test(sum: 0.9999) relative error: 1.0000500025000149E-4")) .build()); } @@ -239,8 +261,8 @@ public void testFloatingPointWithNull() assertEquals( checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), ImmutableMap.builder() - .put(DOUBLE_COLUMN, new ColumnMatchResult(false, "control(sum: 1.0) test(sum: null)")) - .put(REAL_COLUMN, new ColumnMatchResult(false, "control(sum: null) test(sum: 1.0)")) + .put(DOUBLE_COLUMN, new ColumnMatchResult(false, DOUBLE_COLUMN, "control(sum: 1.0) test(sum: null)")) + .put(REAL_COLUMN, new ColumnMatchResult(false, REAL_COLUMN, "control(sum: null) test(sum: 1.0)")) .build()); } @@ -284,8 +306,8 @@ public void testFloatingPointCloseToZero() assertEquals( checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), ImmutableMap.builder() - .put(DOUBLE_COLUMN, new ColumnMatchResult(false, "control(mean: 0.0) test(mean: 1.0199999999999999E-12) difference: 1.0199999999999999E-12")) - .put(REAL_COLUMN, new ColumnMatchResult(false, "control(mean: 1.0199999999999999E-12) test(mean: 0.0) difference: 1.0199999999999999E-12")) + .put(DOUBLE_COLUMN, new ColumnMatchResult(false, DOUBLE_COLUMN, "control(mean: 0.0) test(mean: 1.0199999999999999E-12) difference: 1.0199999999999999E-12")) + .put(REAL_COLUMN, new ColumnMatchResult(false, REAL_COLUMN, "control(mean: 1.0199999999999999E-12) test(mean: 0.0) difference: 1.0199999999999999E-12")) .build()); } @@ -317,8 +339,8 @@ public void testArray() assertEquals( checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), ImmutableMap.builder() - .put(INT_ARRAY_COLUMN, new ColumnMatchResult(false, "control(checksum: 0a, cardinality_sum: 3) test(checksum: 1a, cardinality_sum: 3)")) - .put(MAP_ARRAY_COLUMN, new ColumnMatchResult(false, "control(checksum: 0b, cardinality_sum: 7) test(checksum: 1b, cardinality_sum: 7)")) + .put(INT_ARRAY_COLUMN, new ColumnMatchResult(false, INT_ARRAY_COLUMN, "control(checksum: 0a, cardinality_sum: 3) test(checksum: 1a, cardinality_sum: 3)")) + .put(MAP_ARRAY_COLUMN, new ColumnMatchResult(false, MAP_ARRAY_COLUMN, "control(checksum: 0b, cardinality_sum: 7) test(checksum: 1b, cardinality_sum: 7)")) .build()); // Mismatched different cardinality sum @@ -333,8 +355,8 @@ public void testArray() assertEquals( checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), ImmutableMap.builder() - .put(INT_ARRAY_COLUMN, new ColumnMatchResult(false, "control(checksum: 0a, cardinality_sum: 3) test(checksum: 1a, cardinality_sum: 2)")) - .put(MAP_ARRAY_COLUMN, new ColumnMatchResult(false, "control(checksum: 0b, cardinality_sum: 7) test(checksum: 1b, cardinality_sum: 5)")) + .put(INT_ARRAY_COLUMN, new ColumnMatchResult(false, INT_ARRAY_COLUMN, "control(checksum: 0a, cardinality_sum: 3) test(checksum: 1a, cardinality_sum: 2)")) + .put(MAP_ARRAY_COLUMN, new ColumnMatchResult(false, MAP_ARRAY_COLUMN, "control(checksum: 0b, cardinality_sum: 7) test(checksum: 1b, cardinality_sum: 5)")) .build()); } @@ -342,48 +364,39 @@ public void testArray() public void testRow() { List columns = ImmutableList.of(ROW_COLUMN); - ChecksumResult controlChecksum = new ChecksumResult( - 5, - ImmutableMap.builder() - .put("row_checksum", new SqlVarbinary(new byte[] {0xa})) - .put("row$a_checksum", new SqlVarbinary(new byte[] {0xb})) - .put("row$$col2_checksum", new SqlVarbinary(new byte[] {0xc})) - .put("row$$col3_checksum", new SqlVarbinary(new byte[] {0xd})) - .put("row$b_checksum", new SqlVarbinary(new byte[] {0xe})) - .build()); + ChecksumResult controlChecksum = new ChecksumResult(ROW_COLUMN_CHECKSUMS.size(), ROW_COLUMN_CHECKSUMS); assertTrue(checksumValidator.getMismatchedColumns(columns, controlChecksum, controlChecksum).isEmpty()); // Mismatched different elements ChecksumResult testChecksum = new ChecksumResult( - 5, - ImmutableMap.builder() - .put("row_checksum", new SqlVarbinary(new byte[] {0x1a})) - .put("row$a_checksum", new SqlVarbinary(new byte[] {0x1b})) - .put("row$$col2_checksum", new SqlVarbinary(new byte[] {0x1c})) - .put("row$$col3_checksum", new SqlVarbinary(new byte[] {0x1d})) - .put("row$b_checksum", new SqlVarbinary(new byte[] {0x1e})) - .build()); - assertEquals( - checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), - ImmutableMap.builder() - .put(ROW_COLUMN, new ColumnMatchResult(false, "control(row_checksum: 0a, row$a_checksum: 0b, row$$col2_checksum: 0c, row$$col3_checksum: 0d, row$b_checksum: 0e) test(row_checksum: 1a, row$a_checksum: 1b, row$$col2_checksum: 1c, row$$col3_checksum: 1d, row$b_checksum: 1e)")) - .build()); + ROW_COLUMN_CHECKSUMS.size(), + merge(ROW_COLUMN_CHECKSUMS, ImmutableMap.builder() + .put("row$i_checksum", new SqlVarbinary(new byte[] {0x1a})) + .put("row$r$b_checksum", new SqlVarbinary(new byte[] {0x1d})) + .build())); + + Column aFieldColumn = Column.create("row$i", new DereferenceExpression(ROW_COLUMN.getExpression(), new Identifier("i")), INTEGER); + Column rbFieldColumn = Column.create("row$r$b", new DereferenceExpression(new DereferenceExpression(ROW_COLUMN.getExpression(), new Identifier("r")), new Identifier("b")), BIGINT); - // Mismatched only one elements - testChecksum = new ChecksumResult( - 5, - ImmutableMap.builder() - .put("row_checksum", new SqlVarbinary(new byte[] {0x1a})) - .put("row$a_checksum", new SqlVarbinary(new byte[] {0xb})) - .put("row$$col2_checksum", new SqlVarbinary(new byte[] {0xc})) - .put("row$$col3_checksum", new SqlVarbinary(new byte[] {0x1d})) - .put("row$b_checksum", new SqlVarbinary(new byte[] {0xe})) - .build()); assertEquals( checksumValidator.getMismatchedColumns(columns, controlChecksum, testChecksum), ImmutableMap.builder() - .put(ROW_COLUMN, new ColumnMatchResult(false, "control(row_checksum: 0a, row$$col3_checksum: 0d) test(row_checksum: 1a, row$$col3_checksum: 1d)")) + .put(aFieldColumn, new ColumnMatchResult(false, aFieldColumn, "control(checksum: 0a) test(checksum: 1a)")) + .put(rbFieldColumn, new ColumnMatchResult(false, rbFieldColumn, "control(checksum: 0d) test(checksum: 1d)")) .build()); } + + //ImmutableMap.builder() does not allow overlapping keys + private Map merge(Map origin, Map layer) + { + HashMap result = new HashMap<>(origin); + result.putAll(layer); + return result; + } + + private static Column createColumn(String name, Type type) + { + return Column.create(name, delimitedIdentifier(name), type); + } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java index 853a21ac30f87..249dcabda7b45 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java @@ -18,11 +18,7 @@ import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.tests.StandaloneQueryRunner; import com.facebook.presto.type.TypeRegistry; -import com.facebook.presto.verifier.checksum.ArrayColumnValidator; import com.facebook.presto.verifier.checksum.ChecksumValidator; -import com.facebook.presto.verifier.checksum.FloatingPointColumnValidator; -import com.facebook.presto.verifier.checksum.RowColumnValidator; -import com.facebook.presto.verifier.checksum.SimpleColumnValidator; import com.facebook.presto.verifier.event.DeterminismAnalysisRun; import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; @@ -48,6 +44,7 @@ import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; import static com.facebook.presto.verifier.VerifierTestUtil.CATALOG; import static com.facebook.presto.verifier.VerifierTestUtil.SCHEMA; +import static com.facebook.presto.verifier.VerifierTestUtil.createChecksumValidator; import static com.facebook.presto.verifier.VerifierTestUtil.setupPresto; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.FAILED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; @@ -110,11 +107,7 @@ private DataVerification createVerification(String controlQuery, String testQuer prestoAction, ImmutableList.of(), ImmutableMap.of(CONTROL, QualifiedName.of("tmp_verifier_c"), TEST, QualifiedName.of("tmp_verifier_t"))); - ChecksumValidator checksumValidator = new ChecksumValidator( - new SimpleColumnValidator(), - new FloatingPointColumnValidator(verifierConfig), - new ArrayColumnValidator(), - new RowColumnValidator()); + ChecksumValidator checksumValidator = createChecksumValidator(verifierConfig); SourceQuery sourceQuery = new SourceQuery(SUITE, NAME, controlQuery, testQuery, configuration, configuration); return new DataVerification( (verification, e) -> false, diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java index e4ca778147b9e..0a1cfb2c35cc8 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java @@ -21,11 +21,6 @@ import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.type.TypeRegistry; -import com.facebook.presto.verifier.checksum.ArrayColumnValidator; -import com.facebook.presto.verifier.checksum.ChecksumValidator; -import com.facebook.presto.verifier.checksum.FloatingPointColumnValidator; -import com.facebook.presto.verifier.checksum.RowColumnValidator; -import com.facebook.presto.verifier.checksum.SimpleColumnValidator; import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.prestoaction.NodeResourceClient; import com.facebook.presto.verifier.prestoaction.PrestoAction; @@ -46,6 +41,7 @@ import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.sql.parser.IdentifierSymbol.AT_SIGN; import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; +import static com.facebook.presto.verifier.VerifierTestUtil.createChecksumValidator; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; import static com.facebook.presto.verifier.framework.ClusterType.TEST; @@ -211,11 +207,7 @@ private VerificationManager getVerificationManager(List sourceQueri presto -> new QueryRewriter(SQL_PARSER, presto, ImmutableList.of(), ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX)), new FailureResolverManagerFactory(ImmutableList.of(), new FailureResolverConfig().setEnabled(false)), new MockNodeResourceClient(), - new ChecksumValidator( - new SimpleColumnValidator(), - new FloatingPointColumnValidator(verifierConfig), - new ArrayColumnValidator(), - new RowColumnValidator()), + createChecksumValidator(verifierConfig), verifierConfig, new TypeRegistry(), new FailureResolverConfig().setEnabled(false)),