diff --git a/.travis.yml b/.travis.yml index bd0fe871ad8c8..ae457dd985ebb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ env: - TEST_SPECIFIC_MODULES=presto-accumulo - TEST_SPECIFIC_MODULES=presto-cassandra - TEST_SPECIFIC_MODULES=presto-hive + - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-materialized" - TEST_SPECIFIC_MODULES=presto-main - TEST_SPECIFIC_MODULES=presto-mongodb - TEST_OTHER_MODULES=!presto-tests,!presto-raptor,!presto-accumulo,!presto-cassandra,!presto-hive,!presto-kudu,!presto-docs,!presto-server,!presto-server-rpm,!presto-main,!presto-mongodb diff --git a/pom.xml b/pom.xml index 37e8da0fcc7fb..5ce9cbf550405 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,7 @@ 3.8.0 1.2.3 1.10.0 + 1.6.8 **/TestHiveClientGlueMetastore.java + **/TestHiveDistributedAggregationsWithExchangeMaterialization.java + **/TestHiveDistributedQueriesWithExchangeMaterialization.java + + + + + test-hive-materialized + + - org.basepom.maven - duplicate-finder-maven-plugin + org.apache.maven.plugins + maven-surefire-plugin - - parquet.thrift - about.html - + + **/TestHiveDistributedAggregationsWithExchangeMaterialization.java + **/TestHiveDistributedQueriesWithExchangeMaterialization.java + @@ -374,16 +405,6 @@ - - org.basepom.maven - duplicate-finder-maven-plugin - - - parquet.thrift - about.html - - - diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index ddc71a5a83514..e71702e532e2e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -135,6 +135,7 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; +import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID; import static com.facebook.presto.hive.HivePartitionManager.extractPartitionValues; import static com.facebook.presto.hive.HiveSessionProperties.getHiveStorageFormat; import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableSchema; @@ -407,7 +408,7 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect Iterable> records = () -> stream(partitionManager.getPartitions(metastore, sourceTableHandle, targetConstraint, session).getPartitions()) .map(hivePartition -> - (List) IntStream.range(0, partitionColumns.size()) + IntStream.range(0, partitionColumns.size()) .mapToObj(fieldIdToColumnHandle::get) .map(columnHandle -> hivePartition.getKeys().get(columnHandle).getValue()) .collect(toList())) @@ -1191,25 +1192,27 @@ public Optional finishCreateTable(ConnectorSession sess metastore.createTable(session, table, principalPrivileges, Optional.of(writeInfo.getWritePath()), false, tableStatistics); - if (!handle.getPartitionedBy().isEmpty()) { - if (isRespectTableFormat(session)) { - Verify.verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat()); - } - for (PartitionUpdate update : partitionUpdates) { - Partition partition = buildPartitionObject(session, table, update); - PartitionStatistics partitionStatistics = createPartitionStatistics( - session, - update.getStatistics(), - columnTypes, - getColumnStatistics(partitionComputedStatistics, partition.getValues())); - metastore.addPartition( - session, - handle.getSchemaName(), - handle.getTableName(), - buildPartitionObject(session, table, update), - update.getWritePath(), - partitionStatistics); - } + if (handle.getPartitionedBy().isEmpty()) { + return Optional.of(new HiveWrittenPartitions(ImmutableList.of(UNPARTITIONED_ID))); + } + + if (isRespectTableFormat(session)) { + Verify.verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat()); + } + for (PartitionUpdate update : partitionUpdates) { + Partition partition = buildPartitionObject(session, table, update); + PartitionStatistics partitionStatistics = createPartitionStatistics( + session, + update.getStatistics(), + columnTypes, + getColumnStatistics(partitionComputedStatistics, partition.getValues())); + metastore.addPartition( + session, + handle.getSchemaName(), + handle.getTableName(), + buildPartitionObject(session, table, update), + update.getWritePath(), + partitionStatistics); } return Optional.of(new HiveWrittenPartitions( @@ -1489,6 +1492,7 @@ else if (partitionUpdate.getUpdateMode() == NEW || partitionUpdate.getUpdateMode return Optional.of(new HiveWrittenPartitions( partitionUpdates.stream() .map(PartitionUpdate::getName) + .map(name -> name.isEmpty() ? UNPARTITIONED_ID : name) .collect(Collectors.toList()))); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java index 3eb6c5f6cd8c9..c503af4049f94 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java @@ -494,7 +494,7 @@ else if (insertExistingPartitionsBehavior == InsertExistingPartitionsBehavior.ER outputStorageFormat.getOutputFormat(), writerImplementation, nodeManager.getCurrentNode().getVersion(), - nodeManager.getCurrentNode().getHttpUri().getHost(), + nodeManager.getCurrentNode().getHost(), session.getIdentity().getPrincipal().map(Principal::getName).orElse(null), nodeManager.getEnvironment(), sessionProperties, diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/SubfieldExtractor.java b/presto-hive/src/main/java/com/facebook/presto/hive/SubfieldExtractor.java new file mode 100644 index 0000000000000..4ae125fa21f9e --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/SubfieldExtractor.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.RowType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.DEREFERENCE; +import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static java.util.Collections.unmodifiableList; +import static java.util.Objects.requireNonNull; + +public final class SubfieldExtractor + implements DomainTranslator.ColumnExtractor +{ + private final StandardFunctionResolution functionResolution; + + public SubfieldExtractor(StandardFunctionResolution functionResolution) + { + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + } + + @Override + public Optional extract(RowExpression expression) + { + return toSubfield(expression, functionResolution); + } + + private static Optional toSubfield(RowExpression expression, StandardFunctionResolution functionResolution) + { + List elements = new ArrayList<>(); + while (true) { + if (expression instanceof VariableReferenceExpression) { + Collections.reverse(elements); + return Optional.of(new Subfield(((VariableReferenceExpression) expression).getName(), unmodifiableList(elements))); + } + + if (expression instanceof SpecialFormExpression && ((SpecialFormExpression) expression).getForm() == DEREFERENCE) { + SpecialFormExpression dereferenceExpression = (SpecialFormExpression) expression; + RowExpression base = dereferenceExpression.getArguments().get(0); + RowType baseType = (RowType) base.getType(); + + RowExpression indexExpression = dereferenceExpression.getArguments().get(1); + if (indexExpression instanceof ConstantExpression) { + Object index = ((ConstantExpression) indexExpression).getValue(); + if (index instanceof Number) { + Optional fieldName = baseType.getFields().get(((Number) index).intValue()).getName(); + if (fieldName.isPresent()) { + elements.add(new Subfield.NestedField(fieldName.get())); + expression = base; + continue; + } + } + } + return Optional.empty(); + } + + if (expression instanceof CallExpression && functionResolution.isSubscriptFunction(((CallExpression) expression).getFunctionHandle())) { + List arguments = ((CallExpression) expression).getArguments(); + RowExpression indexExpression = arguments.get(1); + if (indexExpression instanceof ConstantExpression) { + Object index = ((ConstantExpression) indexExpression).getValue(); + if (index instanceof Number) { + elements.add(new Subfield.LongSubscript(((Number) index).longValue())); + expression = arguments.get(0); + continue; + } + + if (isVarcharType(indexExpression.getType())) { + elements.add(new Subfield.StringSubscript(String.valueOf(index))); + expression = arguments.get(0); + continue; + } + } + return Optional.empty(); + } + + return Optional.empty(); + } + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java index 0c65f7fad1c9f..d2c7334a783b9 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.Session; +import com.facebook.presto.execution.QueryManagerConfig.ExchangeMaterializationStrategy; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.file.FileHiveMetastore; @@ -37,6 +38,11 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN; +import static com.facebook.presto.SystemSessionProperties.EXCHANGE_MATERIALIZATION_STRATEGY; +import static com.facebook.presto.SystemSessionProperties.GROUPED_EXECUTION_FOR_AGGREGATION; +import static com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT; +import static com.facebook.presto.SystemSessionProperties.PARTITIONING_PROVIDER_CATALOG; import static com.facebook.presto.spi.security.SelectedRole.Type.ROLE; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; @@ -60,6 +66,7 @@ private HiveQueryRunner() public static final String HIVE_BUCKETED_CATALOG = "hive_bucketed"; public static final String TPCH_SCHEMA = "tpch"; private static final String TPCH_BUCKETED_SCHEMA = "tpch_bucketed"; + private static final String TEMPORARY_TABLE_SCHEMA = "__temporary_tables__"; private static final DateTimeZone TIME_ZONE = DateTimeZone.forID("America/Bahia_Banderas"); public static DistributedQueryRunner createQueryRunner(TpchTable... tables) @@ -112,6 +119,7 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta .put("hive.max-partitions-per-scan", "1000") .put("hive.assume-canonical-partition-keys", "true") .put("hive.collect-column-statistics-on-write", "true") + .put("hive.temporary-table-schema", TEMPORARY_TABLE_SCHEMA) .build(); Map hiveBucketedProperties = ImmutableMap.builder() .putAll(hiveProperties) @@ -133,6 +141,10 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta copyTpchTablesBucketed(queryRunner, "tpch", TINY_SCHEMA_NAME, createBucketedSession(Optional.empty()), tables); } + if (!metastore.getDatabase(TEMPORARY_TABLE_SCHEMA).isPresent()) { + metastore.createDatabase(createDatabaseMetastoreObject(TEMPORARY_TABLE_SCHEMA)); + } + return queryRunner; } catch (Exception e) { @@ -141,6 +153,20 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta } } + public static DistributedQueryRunner createMaterializingQueryRunner(Iterable> tables) + throws Exception + { + return createQueryRunner( + tables, + ImmutableMap.of( + "query.partitioning-provider-catalog", "hive", + "query.exchange-materialization-strategy", "ALL", + "query.hash-partition-count", "11", + "colocated-joins-enabled", "true", + "grouped-execution-for-aggregation-enabled", "true"), + Optional.empty()); + } + private static void setupLogging() { Logging logging = Logging.initialize(); @@ -184,6 +210,24 @@ public static Session createBucketedSession(Optional role) .build(); } + public static Session createMaterializeExchangesSession(Optional role) + { + return testSessionBuilder() + .setIdentity(new Identity( + "hive", + Optional.empty(), + role.map(selectedRole -> ImmutableMap.of("hive", selectedRole)) + .orElse(ImmutableMap.of()))) + .setSystemProperty(PARTITIONING_PROVIDER_CATALOG, HIVE_CATALOG) + .setSystemProperty(EXCHANGE_MATERIALIZATION_STRATEGY, ExchangeMaterializationStrategy.ALL.toString()) + .setSystemProperty(HASH_PARTITION_COUNT, "13") + .setSystemProperty(COLOCATED_JOIN, "true") + .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") + .setCatalog(HIVE_CATALOG) + .setSchema(TPCH_SCHEMA) + .build(); + } + public static void copyTpchTablesBucketed( QueryRunner queryRunner, String sourceCatalog, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestDomainTranslator.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestDomainTranslator.java new file mode 100644 index 0000000000000..967a2ea29b474 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestDomainTranslator.java @@ -0,0 +1,216 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.predicate.Domain; +import com.facebook.presto.spi.predicate.Range; +import com.facebook.presto.spi.predicate.ValueSet; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.DomainTranslator.ExtractionResult; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; + +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; +import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; +import static com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.DEREFERENCE; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IN; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.Expressions.specialForm; +import static com.facebook.presto.sql.relational.LogicalRowExpressions.TRUE; +import static com.facebook.presto.util.Reflection.methodHandle; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.testng.Assert.assertEquals; + +public class TestDomainTranslator +{ + private static final VariableReferenceExpression C_BIGINT = new VariableReferenceExpression("c_bigint", BIGINT); + private static final VariableReferenceExpression C_BIGINT_ARRAY = new VariableReferenceExpression("c_bigint_array", new ArrayType(BIGINT)); + private static final VariableReferenceExpression C_BIGINT_TO_BIGINT_MAP = new VariableReferenceExpression("c_bigint_to_bigint_map", mapType(BIGINT, BIGINT)); + private static final VariableReferenceExpression C_VARCHAR_TO_BIGINT_MAP = new VariableReferenceExpression("c_varchar_to_bigint_map", mapType(VARCHAR, BIGINT)); + private static final VariableReferenceExpression C_STRUCT = new VariableReferenceExpression("c_struct", RowType.from(ImmutableList.of( + RowType.field("a", BIGINT), + RowType.field("b", RowType.from(ImmutableList.of(RowType.field("x", BIGINT)))), + RowType.field("c", new ArrayType(BIGINT)), + RowType.field("d", mapType(BIGINT, BIGINT)), + RowType.field("e", mapType(VARCHAR, BIGINT))))); + + private Metadata metadata; + private RowExpressionDomainTranslator domainTranslator; + private SubfieldExtractor columnExtractor; + + @BeforeClass + public void setup() + { + metadata = createTestMetadataManager(); + domainTranslator = new RowExpressionDomainTranslator(metadata); + columnExtractor = new SubfieldExtractor(new FunctionResolution(metadata.getFunctionManager())); + } + + @Test + public void testSubfields() + { + Map expressions = ImmutableMap.builder() + .put("c_bigint", C_BIGINT) + .put("c_bigint_array[5]", arraySubscript(C_BIGINT_ARRAY, 5)) + .put("c_bigint_to_bigint_map[5]", mapSubscript(C_BIGINT_TO_BIGINT_MAP, constant(5, BIGINT))) + .put("c_varchar_to_bigint_map[\"foo\"]", mapSubscript(C_VARCHAR_TO_BIGINT_MAP, constant("foo", VARCHAR))) + .put("c_struct.a", dereference(C_STRUCT, 0)) + .put("c_struct.b.x", dereference(dereference(C_STRUCT, 1), 0)) + .put("c_struct.c[5]", arraySubscript(dereference(C_STRUCT, 2), 5)) + .put("c_struct.d[5]", mapSubscript(dereference(C_STRUCT, 3), constant(5, BIGINT))) + .put("c_struct.e[\"foo\"]", mapSubscript(dereference(C_STRUCT, 4), constant("foo", VARCHAR))) + .build(); + + for (Map.Entry entry : expressions.entrySet()) { + String subfield = entry.getKey(); + RowExpression expression = entry.getValue(); + + assertPredicateTranslates(greaterThan(expression, bigintLiteral(2L)), subfield, Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 2L)), false)); + + assertPredicateTranslates(equal(expression, bigintLiteral(2L)), subfield, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 2L)), false)); + + assertPredicateTranslates(between(expression, bigintLiteral(1L), bigintLiteral(2L)), subfield, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 1L, true, 2L, true)), false)); + + assertPredicateTranslates(bigintIn(expression, ImmutableList.of(1L)), subfield, Domain.singleValue(BIGINT, 1L)); + + assertPredicateTranslates(bigintIn(expression, ImmutableList.of(1L, 2L)), subfield, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)); + } + } + + private RowExpression dereference(RowExpression base, int field) + { + Type fieldType = base.getType().getTypeParameters().get(field); + return specialForm(DEREFERENCE, fieldType, ImmutableList.of(base, new ConstantExpression(field, INTEGER))); + } + + private RowExpression arraySubscript(RowExpression arrayExpression, int index) + { + ArrayType arrayType = (ArrayType) arrayExpression.getType(); + Type elementType = arrayType.getElementType(); + return call(SUBSCRIPT.name(), + operator(SUBSCRIPT, arrayType, elementType), + elementType, + ImmutableList.of(arrayExpression, constant(index, INTEGER))); + } + + private RowExpression mapSubscript(RowExpression mapExpression, RowExpression keyExpression) + { + MapType mapType = (MapType) mapExpression.getType(); + return call(SUBSCRIPT.name(), + operator(SUBSCRIPT, mapType(mapType.getKeyType(), mapType.getValueType()), mapType.getKeyType()), + mapType.getValueType(), + ImmutableList.of(mapExpression, keyExpression)); + } + + private static MapType mapType(Type keyType, Type valueType) + { + return new MapType( + keyType, + valueType, + methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), + methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), + methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), + methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException")); + } + + public static void throwUnsupportedOperationException() + { + throw new UnsupportedOperationException(); + } + + private FunctionHandle operator(OperatorType operatorType, Type... types) + { + return metadata.getFunctionManager().resolveOperator(operatorType, fromTypes(types)); + } + + private void assertPredicateTranslates(RowExpression predicate, String subfield, Domain domain) + { + ExtractionResult result = domainTranslator.fromPredicate(TEST_SESSION.toConnectorSession(), predicate, columnExtractor); + assertEquals(result.getRemainingExpression(), TRUE); + assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(new Subfield(subfield), domain))); + } + + private RowExpression greaterThan(RowExpression left, RowExpression right) + { + return binaryOperator(OperatorType.GREATER_THAN, left, right); + } + + private RowExpression equal(RowExpression left, RowExpression right) + { + return binaryOperator(OperatorType.EQUAL, left, right); + } + + private RowExpression between(RowExpression value, RowExpression min, RowExpression max) + { + return call( + OperatorType.BETWEEN.name(), + metadata.getFunctionManager().resolveOperator(OperatorType.BETWEEN, fromTypes(value.getType(), min.getType(), max.getType())), + BOOLEAN, + value, + min, + max); + } + + private RowExpression bigintIn(RowExpression value, List inList) + { + List arguments = inList.stream().map(argument -> constant(argument, BIGINT)).collect(toImmutableList()); + return in(value, arguments); + } + + private RowExpression in(RowExpression value, List inList) + { + return new SpecialFormExpression(IN, BOOLEAN, ImmutableList.builder().add(value).addAll(inList).build()); + } + + private RowExpression binaryOperator(OperatorType operatorType, RowExpression left, RowExpression right) + { + return call( + operatorType.name(), + metadata.getFunctionManager().resolveOperator(operatorType, fromTypes(left.getType(), right.getType())), + BOOLEAN, + left, + right); + } + + private RowExpression bigintLiteral(long value) + { + return constant(value, BIGINT); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsHivePartitioning.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java similarity index 61% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsHivePartitioning.java rename to presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java index af85ed9888f94..31253785f56ad 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsHivePartitioning.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java @@ -14,19 +14,24 @@ package com.facebook.presto.hive; import com.facebook.presto.tests.AbstractTestAggregations; -import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; -import java.util.Optional; - -import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; +import static com.facebook.presto.hive.HiveQueryRunner.createMaterializingQueryRunner; +import static com.facebook.presto.hive.TestHiveIntegrationSmokeTest.assertRemoteMaterializedExchangesCount; import static io.airlift.tpch.TpchTable.getTables; -public class TestHiveDistributedAggregationsHivePartitioning +public class TestHiveDistributedAggregationsWithExchangeMaterialization extends AbstractTestAggregations { - public TestHiveDistributedAggregationsHivePartitioning() + public TestHiveDistributedAggregationsWithExchangeMaterialization() + { + super(() -> createMaterializingQueryRunner(getTables())); + } + + @Test + public void testMaterializedExchangesEnabled() { - super(() -> createQueryRunner(getTables(), ImmutableMap.of("query.partitioning-provider-catalog", "hive"), Optional.empty())); + assertQuery(getSession(), "SELECT orderkey, COUNT(*) lines FROM lineitem GROUP BY orderkey", assertRemoteMaterializedExchangesCount(1)); } @Override diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesHivePartitioning.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesHivePartitioning.java deleted file mode 100644 index c0bc657810334..0000000000000 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesHivePartitioning.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.hive; - -import com.facebook.presto.testing.MaterializedResult; -import com.facebook.presto.tests.AbstractTestDistributedQueries; -import com.google.common.collect.ImmutableMap; -import org.testng.annotations.Test; - -import java.util.Optional; - -import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; -import static com.facebook.presto.sql.tree.ExplainType.Type.LOGICAL; -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.tpch.TpchTable.getTables; -import static org.testng.Assert.assertEquals; - -public class TestHiveDistributedQueriesHivePartitioning - extends AbstractTestDistributedQueries -{ - public TestHiveDistributedQueriesHivePartitioning() - { - super(() -> createQueryRunner(getTables(), ImmutableMap.of("query.partitioning-provider-catalog", "hive"), Optional.empty())); - } - - @Override - public void testDelete() - { - // Hive connector currently does not support row-by-row delete - } - - @Override - public void testExcept() - { - // decimal type is not supported by the Hive hash code function - } - - @Override - public void testIntersect() - { - // decimal type is not supported by the Hive hash code function - } - - @Override - public void testNullOnLhsOfInPredicateAllowed() - { - // unknown type is not supported by the Hive hash code function - } - - @Override - public void testQuantifiedComparison() - { - // decimal type is not supported by the Hive hash code function - } - - @Override - public void testSemiJoin() - { - // decimal type is not supported by the Hive hash code function - } - - @Override - public void testUnion() - { - // unknown type is not supported by the Hive hash code function - } - - @Override - public void testUnionRequiringCoercion() - { - // decimal type is not supported by the Hive hash code function - } - - @Override - public void testValues() - { - // decimal type is not supported by the Hive hash code function - } - - @Test - public void testExplainOfCreateTableAs() - { - String query = "CREATE TABLE copy_orders AS SELECT * FROM orders"; - MaterializedResult result = computeActual("EXPLAIN " + query); - assertEquals(getOnlyElement(result.getOnlyColumnAsSet()), getExplainPlan(query, LOGICAL)); - } - - @Override - protected boolean supportsNotNullColumns() - { - return false; - } - - // Hive specific tests should normally go in TestHiveIntegrationSmokeTest -} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java new file mode 100644 index 0000000000000..5b9475e076e6e --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java @@ -0,0 +1,209 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.tests.AbstractTestDistributedQueries; +import org.testng.annotations.Test; + +import static com.facebook.presto.hive.HiveQueryRunner.createMaterializingQueryRunner; +import static com.facebook.presto.hive.TestHiveIntegrationSmokeTest.assertRemoteMaterializedExchangesCount; +import static com.facebook.presto.sql.tree.ExplainType.Type.LOGICAL; +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.airlift.tpch.TpchTable.getTables; +import static org.testng.Assert.assertEquals; + +public class TestHiveDistributedQueriesWithExchangeMaterialization + extends AbstractTestDistributedQueries +{ + public TestHiveDistributedQueriesWithExchangeMaterialization() + { + super(() -> createMaterializingQueryRunner(getTables())); + } + + @Test + public void testMaterializedExchangesEnabled() + { + assertQuery(getSession(), "SELECT orderkey, COUNT(*) lines FROM lineitem GROUP BY orderkey", assertRemoteMaterializedExchangesCount(1)); + } + + @Override + public void testDelete() + { + // Hive connector currently does not support row-by-row delete + } + + @Override + public void testExcept() + { + // decimal type is not supported by the Hive hash code function + } + + @Override + public void testIntersect() + { + // decimal type is not supported by the Hive hash code function + } + + @Override + public void testNullOnLhsOfInPredicateAllowed() + { + // unknown type is not supported by the Hive hash code function + } + + @Override + public void testQuantifiedComparison() + { + // decimal type is not supported by the Hive hash code function + } + + @Override + public void testSemiJoin() + { + // decimal type is not supported by the Hive hash code function + } + + @Override + public void testUnion() + { + // unknown type is not supported by the Hive hash code function + } + + @Override + public void testUnionRequiringCoercion() + { + // decimal type is not supported by the Hive hash code function + } + + @Override + public void testValues() + { + // decimal type is not supported by the Hive hash code function + } + + @Override + public void testAntiJoinNullHandling() + { + // Unsupported Hive type: unknown + } + + @Test + public void testExchangeMaterializationWithConstantFolding() + { + try { + assertUpdate( + // bucket count has to be different from materialized bucket number + "CREATE TABLE test_constant_folding_lineitem_bucketed\n" + + "WITH (bucket_count = 17, bucketed_by = ARRAY['partkey_mod_9', 'partkey', 'suppkey', 'suppkey_varchar']) AS\n" + + "SELECT partkey % 9 partkey_mod_9, partkey, suppkey, CAST(suppkey AS VARCHAR) suppkey_varchar, comment FROM lineitem", + "SELECT count(*) from lineitem"); + assertUpdate( + "CREATE TABLE test_constant_folding_partsupp_unbucketed AS\n" + + "SELECT partkey % 9 partkey_mod_9, partkey, suppkey, CAST(suppkey AS VARCHAR) suppkey_varchar, comment FROM partsupp", + "SELECT count(*) from partsupp"); + + // one constant, third position (suppkey BIGINT) + assertQuery( + getSession(), + "SELECT lineitem.partkey, lineitem.suppkey, lineitem.comment lineitem_comment, partsupp.comment partsupp_comment\n" + + "FROM test_constant_folding_lineitem_bucketed lineitem JOIN test_constant_folding_partsupp_unbucketed partsupp\n" + + "ON\n" + + " lineitem.partkey = partsupp.partkey AND\n" + + " lineitem.partkey_mod_9 = partsupp.partkey_mod_9 AND\n" + + " lineitem.suppkey = partsupp.suppkey AND\n" + + " lineitem.suppkey_varchar = partsupp.suppkey_varchar\n" + + "WHERE lineitem.suppkey = 42", + "SELECT lineitem.partkey, lineitem.suppkey, lineitem.comment lineitem_comment, partsupp.comment partsupp_comment\n" + + "FROM lineitem JOIN partsupp\n" + + "ON lineitem.partkey = partsupp.partkey AND\n" + + "lineitem.suppkey = partsupp.suppkey\n" + + "WHERE lineitem.suppkey = 42", + assertRemoteMaterializedExchangesCount(1)); + + // one constant, fourth position (suppkey_varchar VARCHAR) + assertQuery( + getSession(), + "SELECT lineitem.partkey, lineitem.suppkey, lineitem.comment lineitem_comment, partsupp.comment partsupp_comment\n" + + "FROM test_constant_folding_lineitem_bucketed lineitem JOIN test_constant_folding_partsupp_unbucketed partsupp\n" + + "ON\n" + + " lineitem.partkey = partsupp.partkey AND\n" + + " lineitem.partkey_mod_9 = partsupp.partkey_mod_9 AND\n" + + " lineitem.suppkey = partsupp.suppkey AND\n" + + " lineitem.suppkey_varchar = partsupp.suppkey_varchar\n" + + "WHERE lineitem.suppkey_varchar = '42'", + "SELECT lineitem.partkey, lineitem.suppkey, lineitem.comment lineitem_comment, partsupp.comment partsupp_comment\n" + + "FROM lineitem JOIN partsupp\n" + + "ON lineitem.partkey = partsupp.partkey AND\n" + + "lineitem.suppkey = partsupp.suppkey\n" + + "WHERE lineitem.suppkey = 42", + assertRemoteMaterializedExchangesCount(1)); + + // two constants, first and third position (partkey_mod_9 BIGINT, suppkey BIGINT) + assertQuery( + getSession(), + "SELECT lineitem.partkey, lineitem.suppkey, lineitem.comment lineitem_comment, partsupp.comment partsupp_comment\n" + + "FROM test_constant_folding_lineitem_bucketed lineitem JOIN test_constant_folding_partsupp_unbucketed partsupp\n" + + "ON\n" + + " lineitem.partkey = partsupp.partkey AND\n" + + " lineitem.partkey_mod_9 = partsupp.partkey_mod_9 AND\n" + + " lineitem.suppkey = partsupp.suppkey AND\n" + + " lineitem.suppkey_varchar = partsupp.suppkey_varchar\n" + + "WHERE lineitem.partkey_mod_9 = 7 AND lineitem.suppkey = 42", + "SELECT lineitem.partkey, lineitem.suppkey, lineitem.comment lineitem_comment, partsupp.comment partsupp_comment\n" + + "FROM lineitem JOIN partsupp\n" + + "ON lineitem.partkey = partsupp.partkey AND\n" + + "lineitem.suppkey = partsupp.suppkey\n" + + "WHERE lineitem.partkey % 9 = 7 AND lineitem.suppkey = 42", + assertRemoteMaterializedExchangesCount(1)); + + // two constants, first and forth position (partkey_mod_9 BIGINT, suppkey_varchar VARCHAR) + assertQuery( + getSession(), + "SELECT lineitem.partkey, lineitem.suppkey, lineitem.comment lineitem_comment, partsupp.comment partsupp_comment\n" + + "FROM test_constant_folding_lineitem_bucketed lineitem JOIN test_constant_folding_partsupp_unbucketed partsupp\n" + + "ON\n" + + " lineitem.partkey = partsupp.partkey AND\n" + + " lineitem.partkey_mod_9 = partsupp.partkey_mod_9 AND\n" + + " lineitem.suppkey = partsupp.suppkey AND\n" + + " lineitem.suppkey_varchar = partsupp.suppkey_varchar\n" + + "WHERE lineitem.partkey_mod_9 = 7 AND lineitem.suppkey_varchar = '42'", + "SELECT lineitem.partkey, lineitem.suppkey, lineitem.comment lineitem_comment, partsupp.comment partsupp_comment\n" + + "FROM lineitem JOIN partsupp\n" + + "ON lineitem.partkey = partsupp.partkey AND\n" + + "lineitem.suppkey = partsupp.suppkey\n" + + "WHERE lineitem.partkey % 9 = 7 AND lineitem.suppkey = 42", + assertRemoteMaterializedExchangesCount(1)); + } + finally { + assertUpdate("DROP TABLE IF EXISTS test_constant_folding_lineitem_bucketed"); + assertUpdate("DROP TABLE IF EXISTS test_constant_folding_partsupp_unbucketed"); + } + } + + @Test + public void testExplainOfCreateTableAs() + { + String query = "CREATE TABLE copy_orders AS SELECT * FROM orders"; + MaterializedResult result = computeActual("EXPLAIN " + query); + assertEquals(getOnlyElement(result.getOnlyColumnAsSet()), getExplainPlan(query, LOGICAL)); + } + + @Override + protected boolean supportsNotNullColumns() + { + return false; + } + + // Hive specific tests should normally go in TestHiveIntegrationSmokeTest +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java index 9a6812bf00941..c3c828978cb6d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java @@ -75,6 +75,7 @@ import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveQueryRunner.TPCH_SCHEMA; import static com.facebook.presto.hive.HiveQueryRunner.createBucketedSession; +import static com.facebook.presto.hive.HiveQueryRunner.createMaterializeExchangesSession; import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; import static com.facebook.presto.hive.HiveSessionProperties.RCFILE_OPTIMIZED_WRITER_ENABLED; import static com.facebook.presto.hive.HiveSessionProperties.WRITING_STAGING_FILES_ENABLED; @@ -99,6 +100,7 @@ import static com.facebook.presto.spi.type.VarcharType.createVarcharType; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_MATERIALIZED; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.textLogicalPlan; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; @@ -114,7 +116,9 @@ import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.tpch.TpchTable.CUSTOMER; +import static io.airlift.tpch.TpchTable.LINE_ITEM; import static io.airlift.tpch.TpchTable.ORDERS; +import static io.airlift.tpch.TpchTable.PART_SUPPLIER; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; @@ -134,19 +138,30 @@ public class TestHiveIntegrationSmokeTest { private final String catalog; private final Session bucketedSession; + private final Session materializeExchangesSession; private final TypeTranslator typeTranslator; @SuppressWarnings("unused") public TestHiveIntegrationSmokeTest() { - this(() -> createQueryRunner(ORDERS, CUSTOMER), createBucketedSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin")))), HIVE_CATALOG, new HiveTypeTranslator()); + this(() -> createQueryRunner(ORDERS, CUSTOMER, LINE_ITEM, PART_SUPPLIER), + createBucketedSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin")))), + createMaterializeExchangesSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin")))), + HIVE_CATALOG, + new HiveTypeTranslator()); } - protected TestHiveIntegrationSmokeTest(QueryRunnerSupplier queryRunnerSupplier, Session bucketedSession, String catalog, TypeTranslator typeTranslator) + protected TestHiveIntegrationSmokeTest( + QueryRunnerSupplier queryRunnerSupplier, + Session bucketedSession, + Session materializeExchangesSession, + String catalog, + TypeTranslator typeTranslator) { super(queryRunnerSupplier); this.catalog = requireNonNull(catalog, "catalog is null"); this.bucketedSession = requireNonNull(bucketedSession, "bucketSession is null"); + this.materializeExchangesSession = requireNonNull(materializeExchangesSession, "materializeExchangesSession is null"); this.typeTranslator = requireNonNull(typeTranslator, "typeTranslator is null"); } @@ -2400,6 +2415,12 @@ public void testMismatchedBucketWithBucketPredicate() @Test public void testMismatchedBucketing() + { + testMismatchedBucketing(getSession()); + testMismatchedBucketing(materializeExchangesSession); + } + + public void testMismatchedBucketing(Session session) { try { assertUpdate( @@ -2417,11 +2438,11 @@ public void testMismatchedBucketing() "SELECT orderkey keyN, comment valueN FROM orders", 15000); - Session withMismatchOptimization = Session.builder(getSession()) + Session withMismatchOptimization = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setCatalogSessionProperty(catalog, "optimize_mismatched_bucket_count", "true") .build(); - Session withoutMismatchOptimization = Session.builder(getSession()) + Session withoutMismatchOptimization = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setCatalogSessionProperty(catalog, "optimize_mismatched_bucket_count", "false") .build(); @@ -2470,8 +2491,132 @@ public void testMismatchedBucketing() } } + @Test + public void testMaterializedPartitioning() + { + // Simple smoke tests for materialized partitioning + // Comprehensive testing is done by TestHiveDistributedAggregationsWithExchangeMaterialization, TestHiveDistributedQueriesWithExchangeMaterialization + + // simple aggregation + assertQuery(materializeExchangesSession, "SELECT orderkey, COUNT(*) lines FROM lineitem GROUP BY orderkey", assertRemoteMaterializedExchangesCount(1)); + // simple distinct + assertQuery(materializeExchangesSession, "SELECT distinct orderkey FROM lineitem", assertRemoteMaterializedExchangesCount(1)); + // more complex aggregation + assertQuery(materializeExchangesSession, "SELECT custkey, orderstatus, COUNT(DISTINCT orderkey) FROM orders GROUP BY custkey, orderstatus", assertRemoteMaterializedExchangesCount(2)); + + // join + assertQuery(materializeExchangesSession, "SELECT * FROM (lineitem JOIN orders ON lineitem.orderkey = orders.orderkey) x", assertRemoteMaterializedExchangesCount(2)); + + // 3-way join + try { + assertUpdate("CREATE TABLE test_orders_part1 AS SELECT orderkey, totalprice FROM orders", "SELECT count(*) FROM orders"); + assertUpdate("CREATE TABLE test_orders_part2 AS SELECT orderkey, comment FROM orders", "SELECT count(*) FROM orders"); + + assertQuery( + materializeExchangesSession, + "SELECT lineitem.orderkey, lineitem.comment, test_orders_part1.totalprice, test_orders_part2.comment ordercomment\n" + + "FROM lineitem JOIN test_orders_part1\n" + + "ON lineitem.orderkey = test_orders_part1.orderkey\n" + + "JOIN test_orders_part2\n" + + "ON lineitem.orderkey = test_orders_part2.orderkey", + "SELECT lineitem.orderkey, lineitem.comment, orders.totalprice, orders.comment ordercomment\n" + + "FROM lineitem JOIN orders\n" + + "ON lineitem.orderkey = orders.orderkey", + assertRemoteMaterializedExchangesCount(3)); + } + finally { + assertUpdate("DROP TABLE IF EXISTS test_orders_part1"); + assertUpdate("DROP TABLE IF EXISTS test_orders_part2"); + } + + try { + // join a bucketed table with an unbucketed table + assertUpdate( + // bucket count has to be different from materialized bucket number + "CREATE TABLE test_bucketed_lineitem1\n" + + "WITH (bucket_count = 17, bucketed_by = ARRAY['orderkey']) AS\n" + + "SELECT * FROM lineitem", + "SELECT count(*) from lineitem"); + // bucketed table as probe side + assertQuery( + materializeExchangesSession, + "SELECT * FROM test_bucketed_lineitem1 JOIN orders ON test_bucketed_lineitem1.orderkey = orders.orderkey", + "SELECT * FROM lineitem JOIN orders ON lineitem.orderkey = orders.orderkey", + assertRemoteMaterializedExchangesCount(1)); + // unbucketed table as probe side + assertQuery( + materializeExchangesSession, + "SELECT * FROM orders JOIN test_bucketed_lineitem1 ON test_bucketed_lineitem1.orderkey = orders.orderkey", + "SELECT * FROM orders JOIN lineitem ON lineitem.orderkey = orders.orderkey", + assertRemoteMaterializedExchangesCount(1)); + + // join a bucketed table with an unbucketed table; the join has constant pushdown + assertUpdate( + // bucket count has to be different from materialized bucket number + "CREATE TABLE test_bucketed_lineitem2\n" + + "WITH (bucket_count = 17, bucketed_by = ARRAY['partkey', 'suppkey']) AS\n" + + "SELECT * FROM lineitem", + "SELECT count(*) from lineitem"); + // bucketed table as probe side + assertQuery( + materializeExchangesSession, + "SELECT * \n" + + "FROM test_bucketed_lineitem2 JOIN partsupp\n" + + "ON test_bucketed_lineitem2.partkey = partsupp.partkey AND\n" + + "test_bucketed_lineitem2.suppkey = partsupp.suppkey\n" + + "WHERE test_bucketed_lineitem2.suppkey = 42", + "SELECT * \n" + + "FROM lineitem JOIN partsupp\n" + + "ON lineitem.partkey = partsupp.partkey AND\n" + + "lineitem.suppkey = partsupp.suppkey\n" + + "WHERE lineitem.suppkey = 42", + assertRemoteMaterializedExchangesCount(1)); + // unbucketed table as probe side + assertQuery( + materializeExchangesSession, + "SELECT * \n" + + "FROM partsupp JOIN test_bucketed_lineitem2\n" + + "ON test_bucketed_lineitem2.partkey = partsupp.partkey AND\n" + + "test_bucketed_lineitem2.suppkey = partsupp.suppkey\n" + + "WHERE test_bucketed_lineitem2.suppkey = 42", + "SELECT * \n" + + "FROM partsupp JOIN lineitem\n" + + "ON lineitem.partkey = partsupp.partkey AND\n" + + "lineitem.suppkey = partsupp.suppkey\n" + + "WHERE lineitem.suppkey = 42", + assertRemoteMaterializedExchangesCount(1)); + } + finally { + assertUpdate("DROP TABLE IF EXISTS test_bucketed_lineitem1"); + assertUpdate("DROP TABLE IF EXISTS test_bucketed_lineitem2"); + } + } + + public static Consumer assertRemoteMaterializedExchangesCount(int expectedRemoteExchangesCount) + { + return plan -> + { + int actualRemoteExchangesCount = searchFrom(plan.getRoot()) + .where(node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope() == REMOTE_MATERIALIZED) + .findAll() + .size(); + if (actualRemoteExchangesCount != expectedRemoteExchangesCount) { + throw new AssertionError(format( + "Expected [%s] materialized exchanges but found [%s] materialized exchanges.", + expectedRemoteExchangesCount, + actualRemoteExchangesCount)); + } + }; + } + @Test public void testGroupedExecution() + { + testGroupedExecution(getSession()); + testGroupedExecution(materializeExchangesSession); + } + + public void testGroupedExecution(Session session) { try { assertUpdate( @@ -2510,40 +2655,40 @@ public void testGroupedExecution() 10); // NOT grouped execution; default - Session notColocated = Session.builder(getSession()) + Session notColocated = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "false") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "false") .build(); // Co-located JOIN with all groups at once, fixed schedule - Session colocatedAllGroupsAtOnce = Session.builder(getSession()) + Session colocatedAllGroupsAtOnce = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(CONCURRENT_LIFESPANS_PER_NODE, "0") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "false") .build(); // Co-located JOIN, 1 group per worker at a time, fixed schedule - Session colocatedOneGroupAtATime = Session.builder(getSession()) + Session colocatedOneGroupAtATime = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(CONCURRENT_LIFESPANS_PER_NODE, "1") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "false") .build(); // Co-located JOIN with all groups at once, dynamic schedule - Session colocatedAllGroupsAtOnceDynamic = Session.builder(getSession()) + Session colocatedAllGroupsAtOnceDynamic = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(CONCURRENT_LIFESPANS_PER_NODE, "0") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "true") .build(); // Co-located JOIN, 1 group per worker at a time, dynamic schedule - Session colocatedOneGroupAtATimeDynamic = Session.builder(getSession()) + Session colocatedOneGroupAtATimeDynamic = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(CONCURRENT_LIFESPANS_PER_NODE, "1") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "true") .build(); // Broadcast JOIN, 1 group per worker at a time - Session broadcastOneGroupAtATime = Session.builder(getSession()) + Session broadcastOneGroupAtATime = Session.builder(session) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, BROADCAST.name()) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestSubfieldExtractor.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestSubfieldExtractor.java new file mode 100644 index 0000000000000..bb93996ffe9a5 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestSubfieldExtractor.java @@ -0,0 +1,136 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; +import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.DEREFERENCE; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.Expressions.specialForm; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.facebook.presto.util.Reflection.methodHandle; + +public class TestSubfieldExtractor +{ + private static final VariableReferenceExpression C_BIGINT = new VariableReferenceExpression("c_bigint", BIGINT); + private static final VariableReferenceExpression C_BIGINT_ARRAY = new VariableReferenceExpression("c_bigint_array", new ArrayType(BIGINT)); + private static final VariableReferenceExpression C_BIGINT_TO_BIGINT_MAP = new VariableReferenceExpression("c_bigint_to_bigint_map", mapType(BIGINT, BIGINT)); + private static final VariableReferenceExpression C_VARCHAR_TO_BIGINT_MAP = new VariableReferenceExpression("c_varchar_to_bigint_map", mapType(VARCHAR, BIGINT)); + private static final VariableReferenceExpression C_STRUCT = new VariableReferenceExpression("c_struct", RowType.from(ImmutableList.of( + RowType.field("a", BIGINT), + RowType.field("b", RowType.from(ImmutableList.of(RowType.field("x", BIGINT)))), + RowType.field("c", new ArrayType(BIGINT)), + RowType.field("d", mapType(BIGINT, BIGINT)), + RowType.field("e", mapType(VARCHAR, BIGINT))))); + + private FunctionManager functionManager; + private SubfieldExtractor subfieldExtractor; + + @BeforeClass + public void setup() + { + functionManager = createTestMetadataManager().getFunctionManager(); + subfieldExtractor = new SubfieldExtractor(new FunctionResolution(functionManager)); + } + + @Test + public void test() + { + assertSubfieldExtract(C_BIGINT, "c_bigint"); + assertSubfieldExtract(arraySubscript(C_BIGINT_ARRAY, 5), "c_bigint_array[5]"); + assertSubfieldExtract(mapSubscript(C_BIGINT_TO_BIGINT_MAP, constant(5, BIGINT)), "c_bigint_to_bigint_map[5]"); + assertSubfieldExtract(mapSubscript(C_VARCHAR_TO_BIGINT_MAP, constant("foo", VARCHAR)), "c_varchar_to_bigint_map[\"foo\"]"); + assertSubfieldExtract(dereference(C_STRUCT, 0), "c_struct.a"); + assertSubfieldExtract(dereference(dereference(C_STRUCT, 1), 0), "c_struct.b.x"); + assertSubfieldExtract(arraySubscript(dereference(C_STRUCT, 2), 5), "c_struct.c[5]"); + assertSubfieldExtract(mapSubscript(dereference(C_STRUCT, 3), constant(5, BIGINT)), "c_struct.d[5]"); + assertSubfieldExtract(mapSubscript(dereference(C_STRUCT, 4), constant("foo", VARCHAR)), "c_struct.e[\"foo\"]"); + + assertEquals(subfieldExtractor.extract(constant(2, INTEGER)), Optional.empty()); + } + + private void assertSubfieldExtract(RowExpression expression, String subfield) + { + assertEquals(subfieldExtractor.extract(expression), Optional.of(new Subfield(subfield))); + } + + private RowExpression dereference(RowExpression base, int field) + { + Type fieldType = base.getType().getTypeParameters().get(field); + return specialForm(DEREFERENCE, fieldType, ImmutableList.of(base, new ConstantExpression(field, INTEGER))); + } + + private RowExpression arraySubscript(RowExpression arrayExpression, int index) + { + ArrayType arrayType = (ArrayType) arrayExpression.getType(); + Type elementType = arrayType.getElementType(); + return call(SUBSCRIPT.name(), + operator(SUBSCRIPT, arrayType, elementType), + elementType, + ImmutableList.of(arrayExpression, constant(index, INTEGER))); + } + + private RowExpression mapSubscript(RowExpression mapExpression, RowExpression keyExpression) + { + MapType mapType = (MapType) mapExpression.getType(); + return call(SUBSCRIPT.name(), + operator(SUBSCRIPT, mapType(mapType.getKeyType(), mapType.getValueType()), mapType.getKeyType()), + mapType.getValueType(), + ImmutableList.of(mapExpression, keyExpression)); + } + + private static MapType mapType(Type keyType, Type valueType) + { + return new MapType( + keyType, + valueType, + methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), + methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), + methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), + methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException")); + } + + public static void throwUnsupportedOperationException() + { + throw new UnsupportedOperationException(); + } + + private FunctionHandle operator(OperatorType operatorType, Type... types) + { + return functionManager.resolveOperator(operatorType, fromTypes(types)); + } +} diff --git a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxSplitManager.java b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxSplitManager.java index bc7fcce6827cd..74fca2a720f48 100644 --- a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxSplitManager.java +++ b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxSplitManager.java @@ -14,7 +14,7 @@ package com.facebook.presto.connector.jmx; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; @@ -217,6 +217,6 @@ private static List getAllSplits(ConnectorSplitSource splitSourc private static Node createTestingNode(String hostname) { - return new PrestoNode(hostname, URI.create(format("http://%s:8080", hostname)), NodeVersion.UNKNOWN, false); + return new InternalNode(hostname, URI.create(format("http://%s:8080", hostname)), NodeVersion.UNKNOWN, false); } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java index 96dd87ec5c561..c2851207ba228 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java @@ -48,7 +48,7 @@ public Set getAllNodes() @Override public Set getWorkerNodes() { - return nodeManager.getActiveConnectorNodes(connectorId); + return ImmutableSet.copyOf(nodeManager.getActiveConnectorNodes(connectorId)); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java index ef3c3c0474224..48844f5d2f241 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java @@ -17,6 +17,11 @@ import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.PredicateCompiler; +import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; import static java.util.Objects.requireNonNull; @@ -26,19 +31,34 @@ public class ConnectorContextInstance { private final NodeManager nodeManager; private final TypeManager typeManager; + private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution functionResolution; private final PageSorter pageSorter; private final PageIndexerFactory pageIndexerFactory; + private final RowExpressionService rowExpressionService; + private final DomainTranslator domainTranslator; + private final PredicateCompiler predicateCompiler; public ConnectorContextInstance( NodeManager nodeManager, TypeManager typeManager, + FunctionMetadataManager functionMetadataManager, + StandardFunctionResolution functionResolution, PageSorter pageSorter, - PageIndexerFactory pageIndexerFactory) + PageIndexerFactory pageIndexerFactory, + RowExpressionService rowExpressionService, + DomainTranslator domainTranslator, + PredicateCompiler predicateCompiler) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); + this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); + this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); + this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); } @Override @@ -53,6 +73,18 @@ public TypeManager getTypeManager() return typeManager; } + @Override + public FunctionMetadataManager getFunctionMetadataManager() + { + return functionMetadataManager; + } + + @Override + public StandardFunctionResolution getStandardFunctionResolution() + { + return functionResolution; + } + @Override public PageSorter getPageSorter() { @@ -64,4 +96,22 @@ public PageIndexerFactory getPageIndexerFactory() { return pageIndexerFactory; } + + @Override + public RowExpressionService getRowExpressionService() + { + return rowExpressionService; + } + + @Override + public DomainTranslator getDomainTranslator() + { + return domainTranslator; + } + + @Override + public PredicateCompiler getPredicateCompiler() + { + return predicateCompiler; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index 4f4e9a6435a40..dc0ee155b6275 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -41,6 +41,8 @@ import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.procedure.Procedure; +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.split.PageSinkManager; @@ -48,6 +50,8 @@ import com.facebook.presto.split.RecordPageSourceProvider; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.relational.ConnectorRowExpressionService; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -95,6 +99,8 @@ public class ConnectorManager private final PageIndexerFactory pageIndexerFactory; private final NodeInfo nodeInfo; private final TransactionManager transactionManager; + private final DomainTranslator domainTranslator; + private final PredicateCompiler predicateCompiler; @GuardedBy("this") private final ConcurrentMap connectorFactories = new ConcurrentHashMap<>(); @@ -120,7 +126,9 @@ public ConnectorManager( TypeManager typeManager, PageSorter pageSorter, PageIndexerFactory pageIndexerFactory, - TransactionManager transactionManager) + TransactionManager transactionManager, + DomainTranslator domainTranslator, + PredicateCompiler predicateCompiler) { this.metadataManager = metadataManager; this.catalogManager = catalogManager; @@ -137,6 +145,8 @@ public ConnectorManager( this.pageIndexerFactory = pageIndexerFactory; this.nodeInfo = nodeInfo; this.transactionManager = transactionManager; + this.domainTranslator = domainTranslator; + this.predicateCompiler = predicateCompiler; } @PreDestroy @@ -316,8 +326,13 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact ConnectorContext context = new ConnectorContextInstance( new ConnectorAwareNodeManager(nodeManager, nodeInfo.getEnvironment(), connectorId), typeManager, + metadataManager.getFunctionManager(), + new FunctionResolution(metadataManager.getFunctionManager()), pageSorter, - pageIndexerFactory); + pageIndexerFactory, + new ConnectorRowExpressionService(domainTranslator), + domainTranslator, + predicateCompiler); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(factory.getClass().getClassLoader())) { return factory.create(connectorId.getCatalogName(), properties, context); diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java index deb93783ec7e2..68327e6d2e334 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java @@ -14,13 +14,12 @@ package com.facebook.presto.connector.system; import com.facebook.presto.metadata.AllNodes; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; -import com.facebook.presto.metadata.PrestoNode; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.InMemoryRecordSet; import com.facebook.presto.spi.InMemoryRecordSet.Builder; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.SchemaTableName; @@ -86,22 +85,19 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect return table.build().cursor(); } - private void addRows(Builder table, Set nodes, NodeState state) + private void addRows(Builder table, Set nodes, NodeState state) { - for (Node node : nodes) { - table.addRow(node.getNodeIdentifier(), node.getHttpUri().toString(), getNodeVersion(node), isCoordinator(node), state.toString().toLowerCase(Locale.ENGLISH)); + for (InternalNode node : nodes) { + table.addRow(node.getNodeIdentifier(), node.getInternalUri().toString(), getNodeVersion(node), isCoordinator(node), state.toString().toLowerCase(Locale.ENGLISH)); } } - private static String getNodeVersion(Node node) + private static String getNodeVersion(InternalNode node) { - if (node instanceof PrestoNode) { - return ((PrestoNode) node).getNodeVersion().toString(); - } - return ""; + return node.getNodeVersion().toString(); } - private boolean isCoordinator(Node node) + private boolean isCoordinator(InternalNode node) { return nodeManager.getCoordinators().contains(node); } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplitManager.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplitManager.java index c6eb09863778b..93656fa2b5b26 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplitManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplitManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSession; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.FixedSplitSource; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SystemTable; import com.facebook.presto.spi.SystemTable.Distribution; @@ -72,15 +72,15 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand } ImmutableList.Builder splits = ImmutableList.builder(); - ImmutableSet.Builder nodes = ImmutableSet.builder(); + ImmutableSet.Builder nodes = ImmutableSet.builder(); if (tableDistributionMode == ALL_COORDINATORS) { nodes.addAll(nodeManager.getCoordinators()); } else if (tableDistributionMode == ALL_NODES) { nodes.addAll(nodeManager.getNodes(ACTIVE)); } - Set nodeSet = nodes.build(); - for (Node node : nodeSet) { + Set nodeSet = nodes.build(); + for (InternalNode node : nodeSet) { splits.add(new SystemSplit(tableHandle.getConnectorId(), tableHandle, node.getHostAndPort(), constraint)); } return new FixedSplitSource(splits.build()); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java index bb94750dc20a6..185e4008cd815 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; @@ -37,7 +37,7 @@ import com.facebook.presto.sql.planner.RowExpressionInterpreter; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.AstVisitor; import com.facebook.presto.sql.tree.BetweenPredicate; import com.facebook.presto.sql.tree.BooleanLiteral; @@ -100,7 +100,7 @@ public class FilterStatsCalculator private final ScalarStatsCalculator scalarStatsCalculator; private final StatsNormalizer normalizer; private final LiteralEncoder literalEncoder; - private final StandardFunctionResolution functionResolution; + private final FunctionResolution functionResolution; public FilterStatsCalculator(Metadata metadata, ScalarStatsCalculator scalarStatsCalculator, StatsNormalizer normalizer) { @@ -108,7 +108,7 @@ public FilterStatsCalculator(Metadata metadata, ScalarStatsCalculator scalarStat this.scalarStatsCalculator = requireNonNull(scalarStatsCalculator, "scalarStatsCalculator is null"); this.normalizer = requireNonNull(normalizer, "normalizer is null"); this.literalEncoder = new LiteralEncoder(metadata.getBlockEncodingSerde()); - this.functionResolution = new StandardFunctionResolution(metadata.getFunctionManager()); + this.functionResolution = new FunctionResolution(metadata.getFunctionManager()); } @Deprecated @@ -151,7 +151,7 @@ private Expression simplifyExpression(Session session, Expression predicate, Typ private RowExpression simplifyExpression(Session session, RowExpression predicate) { - RowExpressionInterpreter interpreter = new RowExpressionInterpreter(predicate, metadata, session, true); + RowExpressionInterpreter interpreter = new RowExpressionInterpreter(predicate, metadata, session.toConnectorSession(), true); Object value = interpreter.optimize(); if (value == null) { @@ -513,7 +513,7 @@ public PlanNodeStatsEstimate visitSpecialForm(SpecialFormExpression node, Void c public PlanNodeStatsEstimate visitConstant(ConstantExpression node, Void context) { if (node.getType().equals(BOOLEAN)) { - if ((boolean) node.getValue()) { + if (node.getValue() != null && (boolean) node.getValue()) { return input; } PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.builder(); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 5abe65d9c9429..07ef1fe62b322 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -15,8 +15,8 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; @@ -35,7 +35,7 @@ import com.facebook.presto.sql.planner.RowExpressionInterpreter; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ArithmeticUnaryExpression; import com.facebook.presto.sql.tree.AstVisitor; @@ -68,6 +68,7 @@ import static java.lang.Double.isFinite; import static java.lang.Double.isNaN; import static java.lang.Math.abs; +import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; @@ -97,7 +98,7 @@ private class RowExpressionStatsVisitor { private final PlanNodeStatsEstimate input; private final Session session; - private final StandardFunctionResolution resolution = new StandardFunctionResolution(metadata.getFunctionManager()); + private final FunctionResolution resolution = new FunctionResolution(metadata.getFunctionManager()); public RowExpressionStatsVisitor(PlanNodeStatsEstimate input, Session session) { @@ -121,7 +122,7 @@ public SymbolStatsEstimate visitCall(CallExpression call, Void context) return computeArithmeticBinaryStatistics(call, context); } - Object value = new RowExpressionInterpreter(call, metadata, session, true).optimize(); + Object value = new RowExpressionInterpreter(call, metadata, session.toConnectorSession(), true).optimize(); if (value == null) { return nullStatsEstimate(); @@ -239,7 +240,7 @@ private SymbolStatsEstimate computeNegationStatistics(CallExpression call, Void .setHighValue(-stats.getLowValue()) .build(); } - throw new IllegalStateException("Unexpected sign: " + call.getFunctionHandle().getSignature()); + throw new IllegalStateException(format("Unexpected sign: %s(%s)" + call.getDisplayName(), call.getFunctionHandle())); } private SymbolStatsEstimate computeArithmeticBinaryStatistics(CallExpression call, Void context) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java index 76b16e98bbdc1..afc49a44c435c 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java @@ -26,8 +26,8 @@ import com.facebook.presto.sql.planner.plan.PlanNode; import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.LogicalRowExpressions; -import com.facebook.presto.sql.relational.StandardFunctionResolution; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NotExpression; import com.facebook.presto.sql.tree.SymbolReference; @@ -61,7 +61,7 @@ public class SimpleFilterProjectSemiJoinStatsRule private final FilterStatsCalculator filterStatsCalculator; private final LogicalRowExpressions logicalRowExpressions; - private final StandardFunctionResolution functionResolution; + private final FunctionResolution functionResolution; public SimpleFilterProjectSemiJoinStatsRule(StatsNormalizer normalizer, FilterStatsCalculator filterStatsCalculator, FunctionManager functionManager) { @@ -69,7 +69,7 @@ public SimpleFilterProjectSemiJoinStatsRule(StatsNormalizer normalizer, FilterSt this.filterStatsCalculator = requireNonNull(filterStatsCalculator, "filterStatsCalculator can not be null"); requireNonNull(functionManager, "functionManager can not be null"); this.logicalRowExpressions = new LogicalRowExpressions(functionManager); - this.functionResolution = new StandardFunctionResolution(functionManager); + this.functionResolution = new FunctionResolution(functionManager); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/cost/TaskCountEstimator.java b/presto-main/src/main/java/com/facebook/presto/cost/TaskCountEstimator.java index 9e4eb75851ec2..3f192bea26eb4 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/TaskCountEstimator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/TaskCountEstimator.java @@ -14,8 +14,8 @@ package com.facebook.presto.cost; import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; -import com.facebook.presto.spi.Node; import javax.inject.Inject; @@ -35,7 +35,7 @@ public TaskCountEstimator(NodeSchedulerConfig nodeSchedulerConfig, InternalNodeM requireNonNull(nodeSchedulerConfig, "nodeSchedulerConfig is null"); requireNonNull(nodeManager, "nodeManager is null"); this.numberOfNodes = () -> { - Set activeNodes = nodeManager.getAllNodes().getActiveNodes(); + Set activeNodes = nodeManager.getAllNodes().getActiveNodes(); if (nodeSchedulerConfig.isIncludeCoordinator()) { return activeNodes.size(); } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java index b752be0e0ba8c..7da57cc4438aa 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java @@ -88,7 +88,7 @@ private List getSymbolValues(ValuesNode valuesNode, int symbolId, Sessio if (isExpression(rowExpression)) { return evaluateConstantExpression(castToExpression(rowExpression), symbolType, metadata, session, ImmutableList.of()); } - return evaluateConstantRowExpression(rowExpression, metadata, session); + return evaluateConstantRowExpression(rowExpression, metadata, session.toConnectorSession()); }) .collect(toList()); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/LocationFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/LocationFactory.java index f4e6f227d732b..e61f2b62f6374 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/LocationFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/LocationFactory.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.QueryId; import java.net.URI; @@ -26,7 +26,7 @@ public interface LocationFactory URI createLocalTaskLocation(TaskId taskId); - URI createTaskLocation(Node node, TaskId taskId); + URI createTaskLocation(InternalNode node, TaskId taskId); - URI createMemoryInfoLocation(Node node); + URI createMemoryInfoLocation(InternalNode node); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java index 4906dd941679a..852dc75e0e215 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java @@ -17,12 +17,11 @@ import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.collect.Multimap; -import com.google.common.util.concurrent.ListenableFuture; import java.util.OptionalInt; @@ -43,7 +42,7 @@ public MemoryTrackingRemoteTaskFactory(RemoteTaskFactory remoteTaskFactory, Quer @Override public RemoteTask createRemoteTask(Session session, TaskId taskId, - Node node, + InternalNode node, PlanFragment fragment, Multimap initialSplits, OptionalInt totalPartitions, @@ -65,12 +64,6 @@ public RemoteTask createRemoteTask(Session session, return task; } - @Override - public ListenableFuture removeRemoteSource(TaskId taskId, TaskId remoteSourceTaskId) - { - return remoteTaskFactory.removeRemoteSource(taskId, remoteSourceTaskId); - } - private static final class UpdatePeakMemory implements StateChangeListener { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java b/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java index d472c807c7c76..ac9c55657c120 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.util.FinalizerService; import com.google.common.collect.Sets; import io.airlift.log.Logger; @@ -33,7 +33,7 @@ public class NodeTaskMap { private static final Logger log = Logger.get(NodeTaskMap.class); - private final ConcurrentHashMap nodeTasksMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap nodeTasksMap = new ConcurrentHashMap<>(); private final FinalizerService finalizerService; @Inject @@ -42,22 +42,22 @@ public NodeTaskMap(FinalizerService finalizerService) this.finalizerService = requireNonNull(finalizerService, "finalizerService is null"); } - public void addTask(Node node, RemoteTask task) + public void addTask(InternalNode node, RemoteTask task) { createOrGetNodeTasks(node).addTask(task); } - public int getPartitionedSplitsOnNode(Node node) + public int getPartitionedSplitsOnNode(InternalNode node) { return createOrGetNodeTasks(node).getPartitionedSplitCount(); } - public PartitionedSplitCountTracker createPartitionedSplitCountTracker(Node node, TaskId taskId) + public PartitionedSplitCountTracker createPartitionedSplitCountTracker(InternalNode node, TaskId taskId) { return createOrGetNodeTasks(node).createPartitionedSplitCountTracker(taskId); } - private NodeTasks createOrGetNodeTasks(Node node) + private NodeTasks createOrGetNodeTasks(InternalNode node) { NodeTasks nodeTasks = nodeTasksMap.get(node); if (nodeTasks == null) { @@ -66,7 +66,7 @@ private NodeTasks createOrGetNodeTasks(Node node) return nodeTasks; } - private NodeTasks addNodeTask(Node node) + private NodeTasks addNodeTask(InternalNode node) { NodeTasks newNodeTasks = new NodeTasks(finalizerService); NodeTasks nodeTasks = nodeTasksMap.putIfAbsent(node, newNodeTasks); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTask.java b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTask.java index 45e1f6841554d..07045acdaabfa 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTask.java @@ -40,6 +40,8 @@ public interface RemoteTask void setOutputBuffers(OutputBuffers outputBuffers); + ListenableFuture removeRemoteSource(TaskId remoteSourceTaskId); + /** * Listener is always notified asynchronously using a dedicated notification thread pool so, care should * be taken to avoid leaking {@code this} when adding a listener in a constructor. Additionally, it is diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java index c279974b9cbea..e06c5d6088954 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java @@ -16,12 +16,11 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.collect.Multimap; -import com.google.common.util.concurrent.ListenableFuture; import java.util.OptionalInt; @@ -29,13 +28,11 @@ public interface RemoteTaskFactory { RemoteTask createRemoteTask(Session session, TaskId taskId, - Node node, + InternalNode node, PlanFragment fragment, Multimap initialSplits, OptionalInt totalPartitions, OutputBuffers outputBuffers, PartitionedSplitCountTracker partitionedSplitCountTracker, boolean summarizeTaskInfo); - - ListenableFuture removeRemoteSource(TaskId taskId, TaskId remoteSourceTaskId); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java index 51572cb028d03..409a39e977321 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java @@ -18,9 +18,9 @@ import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.scheduler.SplitSchedulerStats; import com.facebook.presto.failureDetector.FailureDetector; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.RemoteTransactionHandle; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.split.RemoteSplit; import com.facebook.presto.sql.planner.PlanFragment; @@ -80,7 +80,7 @@ public final class SqlStageExecution private final Map exchangeSources; - private final Map> tasks = new ConcurrentHashMap<>(); + private final Map> tasks = new ConcurrentHashMap<>(); @GuardedBy("this") private final AtomicInteger nextTaskId = new AtomicInteger(); @@ -359,7 +359,7 @@ public List getAllTasks() .collect(toImmutableList()); } - public synchronized Optional scheduleTask(Node node, int partition, OptionalInt totalPartitions) + public synchronized Optional scheduleTask(InternalNode node, int partition, OptionalInt totalPartitions) { requireNonNull(node, "node is null"); @@ -370,7 +370,7 @@ public synchronized Optional scheduleTask(Node node, int partition, return Optional.of(scheduleTask(node, new TaskId(stateMachine.getStageId(), partition), ImmutableMultimap.of(), totalPartitions)); } - public synchronized Set scheduleSplits(Node node, Multimap splits, Multimap noMoreSplitsNotification) + public synchronized Set scheduleSplits(InternalNode node, Multimap splits, Multimap noMoreSplitsNotification) { requireNonNull(node, "node is null"); requireNonNull(splits, "splits is null"); @@ -408,7 +408,7 @@ public synchronized Set scheduleSplits(Node node, Multimap sourceSplits, OptionalInt totalPartitions) + private synchronized RemoteTask scheduleTask(InternalNode node, TaskId taskId, Multimap sourceSplits, OptionalInt totalPartitions) { checkArgument(!allTasks.contains(taskId), "A task with id %s already exists", taskId); @@ -456,7 +456,7 @@ private synchronized RemoteTask scheduleTask(Node node, TaskId taskId, Multimap< return task; } - public Set getScheduledNodes() + public Set getScheduledNodes() { return ImmutableSet.copyOf(tasks.keySet()); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/buffer/DiscardingOutputBuffer.java b/presto-main/src/main/java/com/facebook/presto/execution/buffer/DiscardingOutputBuffer.java new file mode 100644 index 0000000000000..2a92c45abaed4 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/buffer/DiscardingOutputBuffer.java @@ -0,0 +1,183 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution.buffer; + +import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.StateMachine; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListenableFuture; +import io.airlift.units.DataSize; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +import static com.facebook.presto.execution.buffer.BufferState.FAILED; +import static com.facebook.presto.execution.buffer.BufferState.FINISHED; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static java.util.Objects.requireNonNull; + +public class DiscardingOutputBuffer + implements OutputBuffer +{ + private static final ListenableFuture NON_BLOCKED = immediateFuture(null); + + private final OutputBuffers outputBuffers; + private final StateMachine state; + + private final AtomicLong totalPagesAdded = new AtomicLong(); + private final AtomicLong totalRowsAdded = new AtomicLong(); + + public DiscardingOutputBuffer(OutputBuffers outputBuffers, StateMachine state) + { + this.outputBuffers = requireNonNull(outputBuffers, "outputBuffers is null"); + this.state = requireNonNull(state, "state is null"); + } + + @Override + public OutputBufferInfo getInfo() + { + BufferState state = this.state.get(); + + return new OutputBufferInfo( + "DISCARD", + state, + state.canAddBuffers(), + state.canAddPages(), + 0, + 0, + totalRowsAdded.get(), + totalPagesAdded.get(), + ImmutableList.of()); + } + + @Override + public boolean isFinished() + { + return state.get() == FINISHED; + } + + @Override + public double getUtilization() + { + return 0; + } + + @Override + public boolean isOverutilized() + { + return false; + } + + @Override + public void addStateChangeListener(StateMachine.StateChangeListener stateChangeListener) + { + state.addStateChangeListener(stateChangeListener); + } + + @Override + public void setOutputBuffers(OutputBuffers newOutputBuffers) + { + requireNonNull(newOutputBuffers, "newOutputBuffers is null"); + + // ignore buffers added after query finishes, which can happen when a query is canceled + // also ignore old versions, which is normal + if (state.get().isTerminal() || outputBuffers.getVersion() >= newOutputBuffers.getVersion()) { + return; + } + + // no more buffers can be added but verify this is valid state change + outputBuffers.checkValidTransition(newOutputBuffers); + } + + @Override + public ListenableFuture get(OutputBuffers.OutputBufferId bufferId, long token, DataSize maxSize) + { + throw new UnsupportedOperationException("DiscardingOutputBuffer must not have any active readers"); + } + + @Override + public void acknowledge(OutputBuffers.OutputBufferId bufferId, long token) + { + throw new UnsupportedOperationException("DiscardingOutputBuffer must not have any active readers"); + } + + @Override + public void abort(OutputBuffers.OutputBufferId bufferId) {} + + @Override + public ListenableFuture isFull() + { + return NON_BLOCKED; + } + + @Override + public void enqueue(Lifespan lifespan, List pages) + { + // update stats + long rowCount = pages.stream().mapToLong(SerializedPage::getPositionCount).sum(); + totalRowsAdded.addAndGet(rowCount); + totalPagesAdded.addAndGet(pages.size()); + } + + @Override + public void enqueue(Lifespan lifespan, int partition, List pages) + { + checkState(partition == 0, "Expected partition number to be zero"); + enqueue(lifespan, pages); + } + + @Override + public void setNoMorePages() + { + state.set(FINISHED); + } + + @Override + public void destroy() + { + state.setIf(FINISHED, oldState -> !oldState.isTerminal()); + } + + @Override + public void fail() + { + state.setIf(FAILED, oldState -> !oldState.isTerminal()); + } + + @Override + public void setNoMorePagesForLifespan(Lifespan lifespan) + { + // NOOP + } + + @Override + public void registerLifespanCompletionCallback(Consumer callback) + { + // NOOP + } + + @Override + public boolean isFinishedForLifespan(Lifespan lifespan) + { + return true; + } + + @Override + public long getPeakMemoryUsage() + { + return 0; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java b/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java index 55dfd515f1cc1..a0c0e48e02238 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java @@ -168,6 +168,9 @@ public void setOutputBuffers(OutputBuffers newOutputBuffers) case ARBITRARY: delegate = new ArbitraryOutputBuffer(taskInstanceId, state, maxBufferSize, systemMemoryContextSupplier, executor); break; + case DISCARDING: + delegate = new DiscardingOutputBuffer(newOutputBuffers, state); + break; } // process pending aborts and reads outside of synchronized lock diff --git a/presto-main/src/main/java/com/facebook/presto/execution/buffer/OutputBuffers.java b/presto-main/src/main/java/com/facebook/presto/execution/buffer/OutputBuffers.java index 8d13ab4702da6..7a7b73472197f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/buffer/OutputBuffers.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/buffer/OutputBuffers.java @@ -26,6 +26,7 @@ import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.ARBITRARY; import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.BROADCAST; +import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.DISCARDING; import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.PARTITIONED; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_BROADCAST_DISTRIBUTION; @@ -38,6 +39,7 @@ public final class OutputBuffers { public static final int BROADCAST_PARTITION_ID = 0; + private static final OutputBuffers DISCARDING_OUTPUT_BUFFERS = createInitialEmptyOutputBuffers(DISCARDING).withNoMoreBufferIds(); public static OutputBuffers createInitialEmptyOutputBuffers(BufferType type) { @@ -59,11 +61,17 @@ else if (partitioningHandle.equals(FIXED_ARBITRARY_DISTRIBUTION)) { return new OutputBuffers(type, 0, false, ImmutableMap.of()); } + public static OutputBuffers createDiscardingOutputBuffers() + { + return DISCARDING_OUTPUT_BUFFERS; + } + public enum BufferType { PARTITIONED, BROADCAST, ARBITRARY, + DISCARDING, } private final BufferType type; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java index 6d16d4952525e..4a8ae95e22450 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java @@ -44,7 +44,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; @@ -98,10 +97,9 @@ static List getPreferredScheduleOrder(Collection f Set rootFragments = fragments.stream() .filter(fragment -> !remoteSources.contains(fragment.getId())) .collect(toImmutableSet()); - checkArgument(rootFragments.size() == 1, "Expected one root fragment, but found: " + rootFragments); Visitor visitor = new Visitor(fragments); - visitor.processFragment(getOnlyElement(rootFragments).getId()); + rootFragments.forEach(fragment -> visitor.processFragment(fragment.getId())); return visitor.getSchedulerOrder(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/BucketNodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/BucketNodeMap.java index 87ef872edad78..e6cf72f41e2ab 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/BucketNodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/BucketNodeMap.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import java.util.Optional; import java.util.function.ToIntFunction; @@ -32,13 +32,13 @@ public BucketNodeMap(ToIntFunction splitToBucket) public abstract int getBucketCount(); - public abstract Optional getAssignedNode(int bucketedId); + public abstract Optional getAssignedNode(int bucketedId); - public abstract void assignBucketToNode(int bucketedId, Node node); + public abstract void assignBucketToNode(int bucketedId, InternalNode node); public abstract boolean isDynamic(); - public final Optional getAssignedNode(Split split) + public final Optional getAssignedNode(Split split) { return getAssignedNode(splitToBucket.applyAsInt(split)); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/DynamicSplitPlacementPolicy.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/DynamicSplitPlacementPolicy.java index 774ba6e23defd..3adaa9705d80c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/DynamicSplitPlacementPolicy.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/DynamicSplitPlacementPolicy.java @@ -14,8 +14,8 @@ package com.facebook.presto.execution.scheduler; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import java.util.List; import java.util.Set; @@ -48,7 +48,7 @@ public void lockDownNodes() } @Override - public List allNodes() + public List allNodes() { return nodeSelector.allNodes(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedBucketNodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedBucketNodeMap.java index 9e977edbe20ae..46858f41255b4 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedBucketNodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedBucketNodeMap.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableList; import java.util.List; @@ -27,16 +27,16 @@ public class FixedBucketNodeMap extends BucketNodeMap { - private final List bucketToNode; + private final List bucketToNode; - public FixedBucketNodeMap(ToIntFunction splitToBucket, List bucketToNode) + public FixedBucketNodeMap(ToIntFunction splitToBucket, List bucketToNode) { super(splitToBucket); this.bucketToNode = ImmutableList.copyOf(requireNonNull(bucketToNode, "bucketToNode is null")); } @Override - public Optional getAssignedNode(int bucketedId) + public Optional getAssignedNode(int bucketedId) { return Optional.of(bucketToNode.get(bucketedId)); } @@ -48,7 +48,7 @@ public int getBucketCount() } @Override - public void assignBucketToNode(int bucketedId, Node node) + public void assignBucketToNode(int bucketedId, InternalNode node) { throw new UnsupportedOperationException(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedCountScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedCountScheduler.java index afafbe61179f0..f458e5cb57b4c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedCountScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedCountScheduler.java @@ -15,7 +15,7 @@ import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.SqlStageExecution; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.google.common.annotations.VisibleForTesting; import java.util.List; @@ -31,13 +31,13 @@ public class FixedCountScheduler { public interface TaskScheduler { - Optional scheduleTask(Node node, int partition, OptionalInt totalPartitions); + Optional scheduleTask(InternalNode node, int partition, OptionalInt totalPartitions); } private final TaskScheduler taskScheduler; - private final List partitionToNode; + private final List partitionToNode; - public FixedCountScheduler(SqlStageExecution stage, List partitionToNode) + public FixedCountScheduler(SqlStageExecution stage, List partitionToNode) { requireNonNull(stage, "stage is null"); this.taskScheduler = stage::scheduleTask; @@ -45,7 +45,7 @@ public FixedCountScheduler(SqlStageExecution stage, List partitionToNode) } @VisibleForTesting - public FixedCountScheduler(TaskScheduler taskScheduler, List partitionToNode) + public FixedCountScheduler(TaskScheduler taskScheduler, List partitionToNode) { this.taskScheduler = requireNonNull(taskScheduler, "taskScheduler is null"); this.partitionToNode = requireNonNull(partitionToNode, "partitionToNode is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java index 82767b021bcfc..75646cbe56b79 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java @@ -20,9 +20,9 @@ import com.facebook.presto.execution.scheduler.group.DynamicLifespanScheduler; import com.facebook.presto.execution.scheduler.group.FixedLifespanScheduler; import com.facebook.presto.execution.scheduler.group.LifespanScheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.StageExecutionDescriptor; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.facebook.presto.split.SplitSource; import com.facebook.presto.sql.planner.plan.PlanNodeId; @@ -57,7 +57,7 @@ public class FixedSourcePartitionedScheduler private static final Logger log = Logger.get(FixedSourcePartitionedScheduler.class); private final SqlStageExecution stage; - private final List nodes; + private final List nodes; private final List sourceSchedulers; private final List partitionHandles; private boolean scheduledTasks; @@ -68,7 +68,7 @@ public FixedSourcePartitionedScheduler( Map splitSources, StageExecutionDescriptor stageExecutionDescriptor, List schedulingOrder, - List nodes, + List nodes, BucketNodeMap bucketNodeMap, int splitBatchSize, OptionalInt concurrentLifespansPerTask, @@ -251,13 +251,13 @@ public static class BucketedSplitPlacementPolicy implements SplitPlacementPolicy { private final NodeSelector nodeSelector; - private final List allNodes; + private final List allNodes; private final BucketNodeMap bucketNodeMap; private final Supplier> remoteTasks; public BucketedSplitPlacementPolicy( NodeSelector nodeSelector, - List allNodes, + List allNodes, BucketNodeMap bucketNodeMap, Supplier> remoteTasks) { @@ -279,12 +279,12 @@ public void lockDownNodes() } @Override - public List allNodes() + public List allNodes() { return allNodes; } - public Node getNodeForBucket(int bucketId) + public InternalNode getNodeForBucket(int bucketId) { return bucketNodeMap.getAssignedNode(bucketId).get(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeAssignmentStats.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeAssignmentStats.java index cecd12fe1b033..023ea9771307a 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeAssignmentStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeAssignmentStats.java @@ -15,7 +15,7 @@ import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import java.util.HashMap; import java.util.List; @@ -27,8 +27,8 @@ public final class NodeAssignmentStats { private final NodeTaskMap nodeTaskMap; - private final Map assignmentCount = new HashMap<>(); - private final Map splitCountByNode = new HashMap<>(); + private final Map assignmentCount = new HashMap<>(); + private final Map splitCountByNode = new HashMap<>(); private final Map queuedSplitCountByNode = new HashMap<>(); public NodeAssignmentStats(NodeTaskMap nodeTaskMap, NodeMap nodeMap, List existingTasks) @@ -36,7 +36,7 @@ public NodeAssignmentStats(NodeTaskMap nodeTaskMap, NodeMap nodeMap, List x + y); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java index bba634b8e62ed..000221b9a4fd3 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.google.common.collect.SetMultimap; import java.net.InetAddress; @@ -22,14 +22,14 @@ public class NodeMap { - private final SetMultimap nodesByHostAndPort; - private final SetMultimap nodesByHost; - private final SetMultimap workersByNetworkPath; + private final SetMultimap nodesByHostAndPort; + private final SetMultimap nodesByHost; + private final SetMultimap workersByNetworkPath; private final Set coordinatorNodeIds; - public NodeMap(SetMultimap nodesByHostAndPort, - SetMultimap nodesByHost, - SetMultimap workersByNetworkPath, + public NodeMap(SetMultimap nodesByHostAndPort, + SetMultimap nodesByHost, + SetMultimap workersByNetworkPath, Set coordinatorNodeIds) { this.nodesByHostAndPort = nodesByHostAndPort; @@ -38,17 +38,17 @@ public NodeMap(SetMultimap nodesByHostAndPort, this.coordinatorNodeIds = coordinatorNodeIds; } - public SetMultimap getNodesByHostAndPort() + public SetMultimap getNodesByHostAndPort() { return nodesByHostAndPort; } - public SetMultimap getNodesByHost() + public SetMultimap getNodesByHost() { return nodesByHost; } - public SetMultimap getWorkersByNetworkPath() + public SetMultimap getWorkersByNetworkPath() { return workersByNetworkPath; } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java index 20d018c36e775..101b873af7c99 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java @@ -16,10 +16,10 @@ import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.HashMultimap; @@ -125,11 +125,11 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) // this supplier is thread-safe. TODO: this logic should probably move to the scheduler since the choice of which node to run in should be // done as close to when the the split is about to be scheduled Supplier nodeMap = Suppliers.memoizeWithExpiration(() -> { - ImmutableSetMultimap.Builder byHostAndPort = ImmutableSetMultimap.builder(); - ImmutableSetMultimap.Builder byHost = ImmutableSetMultimap.builder(); - ImmutableSetMultimap.Builder workersByNetworkPath = ImmutableSetMultimap.builder(); + ImmutableSetMultimap.Builder byHostAndPort = ImmutableSetMultimap.builder(); + ImmutableSetMultimap.Builder byHost = ImmutableSetMultimap.builder(); + ImmutableSetMultimap.Builder workersByNetworkPath = ImmutableSetMultimap.builder(); - Set nodes; + Set nodes; if (connectorId != null) { nodes = nodeManager.getActiveConnectorNodes(connectorId); } @@ -138,10 +138,10 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) } Set coordinatorNodeIds = nodeManager.getCoordinators().stream() - .map(Node::getNodeIdentifier) + .map(InternalNode::getNodeIdentifier) .collect(toImmutableSet()); - for (Node node : nodes) { + for (InternalNode node : nodes) { if (useNetworkTopology && (includeCoordinator || !coordinatorNodeIds.contains(node.getNodeIdentifier()))) { NetworkLocation location = networkLocationCache.get(node.getHostAndPort()); for (int i = 0; i <= location.getSegments().size(); i++) { @@ -151,7 +151,7 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) try { byHostAndPort.put(node.getHostAndPort(), node); - InetAddress host = InetAddress.getByName(node.getHttpUri().getHost()); + InetAddress host = InetAddress.getByName(node.getInternalUri().getHost()); byHost.put(host, node); } catch (UnknownHostException e) { @@ -180,11 +180,11 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) } } - public static List selectNodes(int limit, ResettableRandomizedIterator candidates) + public static List selectNodes(int limit, ResettableRandomizedIterator candidates) { checkArgument(limit > 0, "limit must be at least 1"); - List selected = new ArrayList<>(min(limit, candidates.size())); + List selected = new ArrayList<>(min(limit, candidates.size())); while (selected.size() < limit && candidates.hasNext()) { selected.add(candidates.next()); } @@ -192,18 +192,18 @@ public static List selectNodes(int limit, ResettableRandomizedIterator randomizedNodes(NodeMap nodeMap, boolean includeCoordinator, Set excludedNodes) + public static ResettableRandomizedIterator randomizedNodes(NodeMap nodeMap, boolean includeCoordinator, Set excludedNodes) { - ImmutableList nodes = nodeMap.getNodesByHostAndPort().values().stream() + ImmutableList nodes = nodeMap.getNodesByHostAndPort().values().stream() .filter(node -> includeCoordinator || !nodeMap.getCoordinatorNodeIds().contains(node.getNodeIdentifier())) .filter(node -> !excludedNodes.contains(node)) .collect(toImmutableList()); return new ResettableRandomizedIterator<>(nodes); } - public static List selectExactNodes(NodeMap nodeMap, List hosts, boolean includeCoordinator) + public static List selectExactNodes(NodeMap nodeMap, List hosts, boolean includeCoordinator) { - Set chosen = new LinkedHashSet<>(); + Set chosen = new LinkedHashSet<>(); Set coordinatorIds = nodeMap.getCoordinatorNodeIds(); for (HostAddress host : hosts) { @@ -267,13 +267,13 @@ public static SplitPlacementResult selectDistributionNodes( List existingTasks, BucketNodeMap bucketNodeMap) { - Multimap assignments = HashMultimap.create(); + Multimap assignments = HashMultimap.create(); NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks); - Set blockedNodes = new HashSet<>(); + Set blockedNodes = new HashSet<>(); for (Split split : splits) { // node placement is forced by the bucket to node map - Node node = bucketNodeMap.getAssignedNode(split).get(); + InternalNode node = bucketNodeMap.getAssignedNode(split).get(); // if node is full, don't schedule now, which will push back on the scheduling of splits if (assignmentStats.getTotalSplitCount(node) < maxSplitsPerNode || @@ -295,7 +295,7 @@ public static int calculateLowWatermark(int maxPendingSplitsPerTask) return (int) Math.ceil(maxPendingSplitsPerTask / 2.0); } - public static ListenableFuture toWhenHasSplitQueueSpaceFuture(Set blockedNodes, List existingTasks, int spaceThreshold) + public static ListenableFuture toWhenHasSplitQueueSpaceFuture(Set blockedNodes, List existingTasks, int spaceThreshold) { if (blockedNodes.isEmpty()) { return immediateFuture(null); @@ -305,7 +305,7 @@ public static ListenableFuture toWhenHasSplitQueueSpaceFuture(Set block nodeToTaskMap.put(task.getNodeId(), task); } List> blockedFutures = blockedNodes.stream() - .map(Node::getNodeIdentifier) + .map(InternalNode::getNodeIdentifier) .map(nodeToTaskMap::get) .filter(Objects::nonNull) .map(remoteTask -> remoteTask.whenSplitQueueHasSpace(spaceThreshold)) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSelector.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSelector.java index ae346530a44e3..b58f597313c1f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSelector.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSelector.java @@ -14,8 +14,8 @@ package com.facebook.presto.execution.scheduler; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableSet; import java.util.List; @@ -25,16 +25,16 @@ public interface NodeSelector { void lockDownNodes(); - List allNodes(); + List allNodes(); - Node selectCurrentNode(); + InternalNode selectCurrentNode(); - default List selectRandomNodes(int limit) + default List selectRandomNodes(int limit) { return selectRandomNodes(limit, ImmutableSet.of()); } - List selectRandomNodes(int limit, Set excludedNodes); + List selectRandomNodes(int limit, Set excludedNodes); /** * Identifies the nodes for running the specified splits. diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ScaledWriterScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ScaledWriterScheduler.java index 4b2ba9a56e290..393e4d91a7f2e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ScaledWriterScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ScaledWriterScheduler.java @@ -16,7 +16,7 @@ import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.SqlStageExecution; import com.facebook.presto.execution.TaskStatus; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.SettableFuture; import io.airlift.units.DataSize; @@ -46,7 +46,7 @@ public class ScaledWriterScheduler private final NodeSelector nodeSelector; private final ScheduledExecutorService executor; private final long writerMinSizeBytes; - private final Set scheduledNodes = new HashSet<>(); + private final Set scheduledNodes = new HashSet<>(); private final AtomicBoolean done = new AtomicBoolean(); private volatile SettableFuture future = SettableFuture.create(); @@ -114,12 +114,12 @@ private List scheduleTasks(int count) return ImmutableList.of(); } - List nodes = nodeSelector.selectRandomNodes(count, scheduledNodes); + List nodes = nodeSelector.selectRandomNodes(count, scheduledNodes); checkCondition(!scheduledNodes.isEmpty() || !nodes.isEmpty(), NO_NODES_AVAILABLE, "No nodes available to run query"); ImmutableList.Builder tasks = ImmutableList.builder(); - for (Node node : nodes) { + for (InternalNode node : nodes) { Optional remoteTask = stage.scheduleTask(node, scheduledNodes.size(), OptionalInt.empty()); remoteTask.ifPresent(task -> { tasks.add(task); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java index 1b4d78588aaf3..47d5b22b9f653 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java @@ -15,9 +15,9 @@ import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -80,20 +80,20 @@ public void lockDownNodes() } @Override - public List allNodes() + public List allNodes() { return ImmutableList.copyOf(nodeMap.get().get().getNodesByHostAndPort().values()); } @Override - public Node selectCurrentNode() + public InternalNode selectCurrentNode() { // TODO: this is a hack to force scheduling on the coordinator return nodeManager.getCurrentNode(); } @Override - public List selectRandomNodes(int limit, Set excludedNodes) + public List selectRandomNodes(int limit, Set excludedNodes) { return selectNodes(limit, randomizedNodes(nodeMap.get().get(), includeCoordinator, excludedNodes)); } @@ -101,17 +101,17 @@ public List selectRandomNodes(int limit, Set excludedNodes) @Override public SplitPlacementResult computeAssignments(Set splits, List existingTasks) { - Multimap assignment = HashMultimap.create(); + Multimap assignment = HashMultimap.create(); NodeMap nodeMap = this.nodeMap.get().get(); NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks); - ResettableRandomizedIterator randomCandidates = randomizedNodes(nodeMap, includeCoordinator, ImmutableSet.of()); - Set blockedExactNodes = new HashSet<>(); + ResettableRandomizedIterator randomCandidates = randomizedNodes(nodeMap, includeCoordinator, ImmutableSet.of()); + Set blockedExactNodes = new HashSet<>(); boolean splitWaitingForAnyNode = false; for (Split split : splits) { randomCandidates.reset(); - List candidateNodes; + List candidateNodes; if (!split.isRemotelyAccessible()) { candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator); } @@ -123,10 +123,10 @@ public SplitPlacementResult computeAssignments(Set splits, List splits, List splitAssignment = ImmutableMultimap.of(); + Multimap splitAssignment = ImmutableMultimap.of(); if (!pendingSplits.isEmpty()) { if (!scheduleGroup.placementFuture.isDone()) { anyBlockedOnPlacements = true; @@ -272,11 +272,11 @@ else if (pendingSplits.isEmpty()) { } // if no new splits will be assigned, update state and attach completion event - Multimap noMoreSplitsNotification = ImmutableMultimap.of(); + Multimap noMoreSplitsNotification = ImmutableMultimap.of(); if (pendingSplits.isEmpty() && scheduleGroup.state == ScheduleGroupState.NO_MORE_SPLITS) { scheduleGroup.state = ScheduleGroupState.DONE; if (!lifespan.isTaskWide()) { - Node node = ((BucketedSplitPlacementPolicy) splitPlacementPolicy).getNodeForBucket(lifespan.getId()); + InternalNode node = ((BucketedSplitPlacementPolicy) splitPlacementPolicy).getNodeForBucket(lifespan.getId()); noMoreSplitsNotification = ImmutableMultimap.of(node, lifespan); } } @@ -429,15 +429,15 @@ public synchronized void notifyAllLifespansFinishedExecution() whenFinishedOrNewLifespanAdded.set(null); } - private Set assignSplits(Multimap splitAssignment, Multimap noMoreSplitsNotification) + private Set assignSplits(Multimap splitAssignment, Multimap noMoreSplitsNotification) { ImmutableSet.Builder newTasks = ImmutableSet.builder(); - ImmutableSet nodes = ImmutableSet.builder() + ImmutableSet nodes = ImmutableSet.builder() .addAll(splitAssignment.keySet()) .addAll(noMoreSplitsNotification.keySet()) .build(); - for (Node node : nodes) { + for (InternalNode node : nodes) { // source partitioned tasks can only receive broadcast data; otherwise it would have a different distribution ImmutableMultimap splits = ImmutableMultimap.builder() .putAll(partitionedNode, splitAssignment.get(node)) @@ -463,7 +463,7 @@ private Set finalizeTaskCreationIfNecessary() splitPlacementPolicy.lockDownNodes(); - Set scheduledNodes = stage.getScheduledNodes(); + Set scheduledNodes = stage.getScheduledNodes(); Set newTasks = splitPlacementPolicy.allNodes().stream() .filter(node -> !scheduledNodes.contains(node)) .flatMap(node -> stage.scheduleSplits(node, ImmutableMultimap.of(), ImmutableMultimap.of()).stream()) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementPolicy.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementPolicy.java index 8c33676d5207f..9e8a9bf718791 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementPolicy.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementPolicy.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import java.util.List; import java.util.Set; @@ -25,5 +25,5 @@ public interface SplitPlacementPolicy void lockDownNodes(); - List allNodes(); + List allNodes(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementResult.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementResult.java index 0ce5fb1e0a8a3..2399b694fe86d 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementResult.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementResult.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListenableFuture; @@ -23,9 +23,9 @@ public final class SplitPlacementResult { private final ListenableFuture blocked; - private final Multimap assignments; + private final Multimap assignments; - public SplitPlacementResult(ListenableFuture blocked, Multimap assignments) + public SplitPlacementResult(ListenableFuture blocked, Multimap assignments) { this.blocked = requireNonNull(blocked, "blocked is null"); this.assignments = requireNonNull(assignments, "assignments is null"); @@ -36,7 +36,7 @@ public ListenableFuture getBlocked() return blocked; } - public Multimap getAssignments() + public Multimap getAssignments() { return assignments; } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java index f29be2a97af1c..6df2db1a0d2e5 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java @@ -31,17 +31,19 @@ import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; import com.facebook.presto.failureDetector.FailureDetector; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.facebook.presto.split.SplitSource; import com.facebook.presto.sql.planner.NodePartitionMap; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.PartitioningHandle; +import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.SplitSourceFactory; import com.facebook.presto.sql.planner.SubPlan; import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.facebook.presto.sql.planner.plan.PlanNodeId; +import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -80,8 +82,10 @@ import static com.facebook.presto.execution.StageState.CANCELED; import static com.facebook.presto.execution.StageState.FAILED; import static com.facebook.presto.execution.StageState.FINISHED; +import static com.facebook.presto.execution.StageState.PLANNED; import static com.facebook.presto.execution.StageState.RUNNING; import static com.facebook.presto.execution.StageState.SCHEDULED; +import static com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers; import static com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; @@ -98,6 +102,8 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.newConcurrentHashSet; +import static com.google.common.collect.Streams.stream; +import static com.google.common.graph.Traverser.forTree; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static io.airlift.concurrent.MoreFutures.whenAnyComplete; @@ -114,6 +120,8 @@ public class SqlQueryScheduler { private final QueryStateMachine queryStateMachine; private final ExecutionPolicy executionPolicy; + private final SubPlan plan; + private final StreamingPlanSection sectionedPlan; private final Map stages; private final ExecutorService executor; private final StageId rootStageId; @@ -122,6 +130,7 @@ public class SqlQueryScheduler private final SplitSchedulerStats schedulerStats; private final boolean summarizeTaskInfo; private final AtomicBoolean started = new AtomicBoolean(); + private final AtomicBoolean scheduling = new AtomicBoolean(); public static SqlQueryScheduler createSqlQueryScheduler( QueryStateMachine queryStateMachine, @@ -184,6 +193,7 @@ private SqlQueryScheduler( SplitSchedulerStats schedulerStats) { this.queryStateMachine = requireNonNull(queryStateMachine, "queryStateMachine is null"); + this.plan = requireNonNull(plan, "plan is null"); this.executionPolicy = requireNonNull(executionPolicy, "schedulerPolicyFactory is null"); this.schedulerStats = requireNonNull(schedulerStats, "schedulerStats is null"); this.summarizeTaskInfo = summarizeTaskInfo; @@ -192,20 +202,19 @@ private SqlQueryScheduler( ImmutableMap.Builder stageSchedulers = ImmutableMap.builder(); ImmutableMap.Builder stageLinkages = ImmutableMap.builder(); - // Only fetch a distribution once per query to assure all stages see the same machine assignments - Map partitioningCache = new HashMap<>(); - OutputBufferId rootBufferId = Iterables.getOnlyElement(rootOutputBuffers.getBuffers().keySet()); + sectionedPlan = extractStreamingSections(plan); List stages = createStages( (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations), locationFactory, - plan.withBucketToPartition(Optional.of(new int[1])), + sectionedPlan, + Optional.of(new int[1]), + rootOutputBuffers, nodeScheduler, remoteTaskFactory, splitSourceFactory, session, splitBatchSize, - partitioningHandle -> partitioningCache.computeIfAbsent(partitioningHandle, handle -> nodePartitioningManager.getNodePartitioningMap(session, handle)), nodePartitioningManager, queryExecutor, schedulerExecutor, @@ -214,9 +223,7 @@ private SqlQueryScheduler( stageSchedulers, stageLinkages); - SqlStageExecution rootStage = stages.get(0); - rootStage.setOutputBuffers(rootOutputBuffers); - this.rootStageId = rootStage.getStageId(); + this.rootStageId = stages.get(0).getStageId(); this.stages = stages.stream() .collect(toImmutableMap(SqlStageExecution::getStageId, identity())); @@ -253,6 +260,12 @@ else if (state == ABORTED) { // this should never happen, since abort can only be triggered in query clean up after the query is finished queryStateMachine.transitionToFailed(new PrestoException(GENERIC_INTERNAL_ERROR, "Query stage was aborted")); } + else if (state == FINISHED) { + List nextStagesToSchedule = getStagesReadyForExecution(); + if (!nextStagesToSchedule.isEmpty()) { + startScheduling(nextStagesToSchedule); + } + } else if (queryStateMachine.getQueryState() == QueryState.STARTING) { // if the stage has at least one task, we are running if (stage.hasTasks()) { @@ -289,9 +302,76 @@ private static URI getBufferLocation(RemoteTask remoteTask, OutputBufferId rootB } private List createStages( + ExchangeLocationsConsumer locationsConsumer, + LocationFactory locationFactory, + StreamingPlanSection section, + Optional bucketToPartition, + OutputBuffers outputBuffers, + NodeScheduler nodeScheduler, + RemoteTaskFactory remoteTaskFactory, + SplitSourceFactory splitSourceFactory, + Session session, + int splitBatchSize, + NodePartitioningManager nodePartitioningManager, + ExecutorService queryExecutor, + ScheduledExecutorService schedulerExecutor, + FailureDetector failureDetector, + NodeTaskMap nodeTaskMap, + ImmutableMap.Builder stageSchedulers, + ImmutableMap.Builder stageLinkages) + { + ImmutableList.Builder stages = ImmutableList.builder(); + + // Only fetch a distribution once per section to ensure all stages see the same machine assignments + Map partitioningCache = new HashMap<>(); + List sectionStages = createStreamingLinkedStages( + locationsConsumer, + locationFactory, + section.getPlan().withBucketToPartition(bucketToPartition), + nodeScheduler, + remoteTaskFactory, + splitSourceFactory, + session, + splitBatchSize, + partitioningHandle -> partitioningCache.computeIfAbsent(partitioningHandle, handle -> nodePartitioningManager.getNodePartitioningMap(session, handle)), + nodePartitioningManager, + queryExecutor, + schedulerExecutor, + failureDetector, + nodeTaskMap, + stageSchedulers, + stageLinkages); + sectionStages.get(0).setOutputBuffers(outputBuffers); + stages.addAll(sectionStages); + + for (StreamingPlanSection childSection : section.getChildren()) { + stages.addAll(createStages( + discardingLocationConsumer(), + locationFactory, + childSection, + Optional.empty(), + createDiscardingOutputBuffers(), + nodeScheduler, + remoteTaskFactory, + splitSourceFactory, + session, + splitBatchSize, + nodePartitioningManager, + queryExecutor, + schedulerExecutor, + failureDetector, + nodeTaskMap, + stageSchedulers, + stageLinkages)); + } + + return stages.build(); + } + + private List createStreamingLinkedStages( ExchangeLocationsConsumer parent, LocationFactory locationFactory, - SubPlan plan, + StreamingSubPlan plan, NodeScheduler nodeScheduler, RemoteTaskFactory remoteTaskFactory, SplitSourceFactory splitSourceFactory, @@ -309,7 +389,7 @@ private List createStages( ImmutableList.Builder stages = ImmutableList.builder(); PlanFragmentId fragmentId = plan.getFragment().getId(); - StageId stageId = new StageId(queryStateMachine.getQueryId(), fragmentId.getId()); + StageId stageId = getStageId(fragmentId); SqlStageExecution stage = createSqlStageExecution( stageId, locationFactory.createStageLocation(stageId), @@ -365,7 +445,7 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { } BucketNodeMap bucketNodeMap; - List stageNodeList; + List stageNodeList; if (plan.getFragment().getRemoteSourceNodes().stream().allMatch(node -> node.getExchangeType() == REPLICATE)) { // no non-replicated remote source boolean dynamicLifespanSchedule = plan.getFragment().getStageExecutionDescriptor().isDynamicLifespanSchedule(); @@ -407,7 +487,7 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { else { // all sources are remote NodePartitionMap nodePartitionMap = partitioningCache.apply(plan.getFragment().getPartitioning()); - List partitionToNode = nodePartitionMap.getPartitionToNode(); + List partitionToNode = nodePartitionMap.getPartitionToNode(); // todo this should asynchronously wait a standard timeout period before failing checkCondition(!partitionToNode.isEmpty(), NO_NODES_AVAILABLE, "No worker nodes available"); stageSchedulers.put(stageId, new FixedCountScheduler(stage, partitionToNode)); @@ -416,11 +496,11 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { } ImmutableSet.Builder childStagesBuilder = ImmutableSet.builder(); - for (SubPlan subStagePlan : plan.getChildren()) { - List subTree = createStages( + for (StreamingSubPlan stagePlan : plan.getChildren()) { + List subTree = createStreamingLinkedStages( stage::addExchangeLocations, locationFactory, - subStagePlan.withBucketToPartition(bucketToPartition), + stagePlan.withBucketToPartition(bucketToPartition), nodeScheduler, remoteTaskFactory, splitSourceFactory, @@ -489,29 +569,28 @@ public StageInfo getStageInfo() .map(SqlStageExecution::getStageInfo) .collect(toImmutableMap(StageInfo::getStageId, identity())); - return buildStageInfo(rootStageId, stageInfos); + return buildStageInfo(plan, stageInfos); } - private StageInfo buildStageInfo(StageId stageId, Map stageInfos) + private StageInfo buildStageInfo(SubPlan subPlan, Map stageInfos) { - StageInfo parent = stageInfos.get(stageId); - checkArgument(parent != null, "No stageInfo for %s", parent); - List childStages = stageLinkages.get(stageId).getChildStageIds().stream() - .map(childStageId -> buildStageInfo(childStageId, stageInfos)) - .collect(toImmutableList()); - if (childStages.isEmpty()) { - return parent; + StageInfo stageInfo = stageInfos.get(getStageId(subPlan.getFragment().getId())); + checkArgument(stageInfo != null, "No stageInfo for %s", stageInfo); + if (subPlan.getChildren().isEmpty()) { + return stageInfo; } return new StageInfo( - parent.getStageId(), - parent.getState(), - parent.getSelf(), - parent.getPlan(), - parent.getTypes(), - parent.getStageStats(), - parent.getTasks(), - childStages, - parent.getFailureCause()); + stageInfo.getStageId(), + stageInfo.getState(), + stageInfo.getSelf(), + stageInfo.getPlan(), + stageInfo.getTypes(), + stageInfo.getStageStats(), + stageInfo.getTasks(), + subPlan.getChildren().stream() + .map(plan -> buildStageInfo(plan, stageInfos)) + .collect(toImmutableList()), + stageInfo.getFailureCause()); } public long getUserMemoryReservation() @@ -539,15 +618,69 @@ public Duration getTotalCpuTime() public void start() { if (started.compareAndSet(false, true)) { - executor.submit(this::schedule); + startScheduling(getStagesReadyForExecution()); } } - private void schedule() + private List getStagesReadyForExecution() + { + return stream(forTree(StreamingPlanSection::getChildren).depthFirstPreOrder(sectionedPlan)) + // get all sections ready for execution + .filter(this::isReadyForExecution) + // get all stages in the sections + .flatMap(section -> stream(forTree(StreamingSubPlan::getChildren).depthFirstPreOrder(section.getPlan()))) + .map(StreamingSubPlan::getFragment) + .map(PlanFragment::getId) + .map(this::getStageExecution) + .collect(toImmutableList()); + } + + private boolean isReadyForExecution(StreamingPlanSection section) { + SqlStageExecution stage = getStageExecution(section.getPlan().getFragment().getId()); + if (stage.getState() != PLANNED) { + // already scheduled + return false; + } + for (StreamingPlanSection child : section.getChildren()) { + SqlStageExecution childRootStage = getStageExecution(child.getPlan().getFragment().getId()); + if (childRootStage.getState() != FINISHED) { + return false; + } + } + return true; + } + + private SqlStageExecution getStageExecution(PlanFragmentId planFragmentId) + { + return stages.get(getStageId(planFragmentId)); + } + + private StageId getStageId(PlanFragmentId fragmentId) + { + return new StageId(queryStateMachine.getQueryId(), fragmentId.getId()); + } + + private void startScheduling(Collection stages) + { + requireNonNull(stages); + // still scheduling the previous batch of stages + if (scheduling.get()) { + return; + } + executor.submit(() -> schedule(stages)); + } + + private void schedule(Collection stages) + { + if (!scheduling.compareAndSet(false, true)) { + // still scheduling the previous batch of stages + return; + } + try (SetThreadName ignored = new SetThreadName("Query-%s", queryStateMachine.getQueryId())) { Set completedStages = new HashSet<>(); - ExecutionSchedule executionSchedule = executionPolicy.createExecutionSchedule(stages.values()); + ExecutionSchedule executionSchedule = executionPolicy.createExecutionSchedule(stages); while (!executionSchedule.isFinished()) { List> blockedStages = new ArrayList<>(); for (SqlStageExecution stage : executionSchedule.getStagesToSchedule()) { @@ -591,7 +724,7 @@ else if (!result.getBlocked().isDone()) { } // make sure to update stage linkage at least once per loop to catch async state changes (e.g., partial cancel) - for (SqlStageExecution stage : stages.values()) { + for (SqlStageExecution stage : stages) { if (!completedStages.contains(stage.getStageId()) && stage.getState().isDone()) { stageLinkages.get(stage.getStageId()) .processScheduleResults(stage.getState(), ImmutableSet.of()); @@ -610,22 +743,29 @@ else if (!result.getBlocked().isDone()) { } } - for (SqlStageExecution stage : stages.values()) { + for (SqlStageExecution stage : stages) { StageState state = stage.getState(); if (state != SCHEDULED && state != RUNNING && !state.isDone()) { throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Scheduling is complete, but stage %s is in state %s", stage.getStageId(), state)); } } + + scheduling.set(false); + List nextStagesToSchedule = getStagesReadyForExecution(); + if (!nextStagesToSchedule.isEmpty()) { + startScheduling(nextStagesToSchedule); + } } catch (Throwable t) { queryStateMachine.transitionToFailed(t); throw t; } finally { + scheduling.set(false); RuntimeException closeError = new RuntimeException(); - for (StageScheduler scheduler : stageSchedulers.values()) { + for (SqlStageExecution stage : stages) { try { - scheduler.close(); + stageSchedulers.get(stage.getStageId()).close(); } catch (Throwable t) { queryStateMachine.transitionToFailed(t); @@ -676,11 +816,45 @@ private static ListenableFuture whenAllStages(Collection s return future; } + public static StreamingPlanSection extractStreamingSections(SubPlan subPlan) + { + ImmutableList.Builder materializedExchangeChildren = ImmutableList.builder(); + StreamingSubPlan streamingSection = extractStreamingSection(subPlan, materializedExchangeChildren); + return new StreamingPlanSection( + streamingSection, + materializedExchangeChildren.build().stream() + .map(SqlQueryScheduler::extractStreamingSections) + .collect(toImmutableList())); + } + + private static StreamingSubPlan extractStreamingSection(SubPlan subPlan, ImmutableList.Builder materializedExchangeChildren) + { + ImmutableList.Builder streamingSources = ImmutableList.builder(); + Set streamingFragmentIds = subPlan.getFragment().getRemoteSourceNodes().stream() + .map(RemoteSourceNode::getSourceFragmentIds) + .flatMap(List::stream) + .collect(toImmutableSet()); + for (SubPlan child : subPlan.getChildren()) { + if (streamingFragmentIds.contains(child.getFragment().getId())) { + streamingSources.add(extractStreamingSection(child, materializedExchangeChildren)); + } + else { + materializedExchangeChildren.add(child); + } + } + return new StreamingSubPlan(subPlan.getFragment(), streamingSources.build()); + } + private interface ExchangeLocationsConsumer { void addExchangeLocations(PlanFragmentId fragmentId, Set tasks, boolean noMoreExchangeLocations); } + private static ExchangeLocationsConsumer discardingLocationConsumer() + { + return (fragmentId, tasks, noMoreExchangeLocations) -> {}; + } + private static class StageLinkage { private final PlanFragmentId currentStageFragmentId; @@ -755,4 +929,58 @@ public void processScheduleResults(StageState newState, Set newTasks } } } + + private static class StreamingPlanSection + { + private final StreamingSubPlan plan; + // materialized exchange children + private final List children; + + public StreamingPlanSection(StreamingSubPlan plan, List children) + { + this.plan = requireNonNull(plan, "plan is null"); + this.children = ImmutableList.copyOf(requireNonNull(children, "children is null")); + } + + public StreamingSubPlan getPlan() + { + return plan; + } + + public List getChildren() + { + return children; + } + } + + /** + * StreamingSubPlan is similar to SubPlan but only contains streaming children + */ + private static class StreamingSubPlan + { + private final PlanFragment fragment; + // streaming children + private final List children; + + public StreamingSubPlan(PlanFragment fragment, List children) + { + this.fragment = requireNonNull(fragment, "fragment is null"); + this.children = ImmutableList.copyOf(requireNonNull(children, "children is null")); + } + + public PlanFragment getFragment() + { + return fragment; + } + + public List getChildren() + { + return children; + } + + public StreamingSubPlan withBucketToPartition(Optional bucketToPartition) + { + return new StreamingSubPlan(fragment.withBucketToPartition(bucketToPartition), children); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java index 34c94fee1a7d2..0b4a1fd1b2122 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java @@ -15,10 +15,10 @@ import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -94,20 +94,20 @@ public void lockDownNodes() } @Override - public List allNodes() + public List allNodes() { return ImmutableList.copyOf(nodeMap.get().get().getNodesByHostAndPort().values()); } @Override - public Node selectCurrentNode() + public InternalNode selectCurrentNode() { // TODO: this is a hack to force scheduling on the coordinator return nodeManager.getCurrentNode(); } @Override - public List selectRandomNodes(int limit, Set excludedNodes) + public List selectRandomNodes(int limit, Set excludedNodes) { return selectNodes(limit, randomizedNodes(nodeMap.get().get(), includeCoordinator, excludedNodes)); } @@ -116,21 +116,21 @@ public List selectRandomNodes(int limit, Set excludedNodes) public SplitPlacementResult computeAssignments(Set splits, List existingTasks) { NodeMap nodeMap = this.nodeMap.get().get(); - Multimap assignment = HashMultimap.create(); + Multimap assignment = HashMultimap.create(); NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks); int[] topologicCounters = new int[topologicalSplitCounters.size()]; Set filledLocations = new HashSet<>(); - Set blockedExactNodes = new HashSet<>(); + Set blockedExactNodes = new HashSet<>(); boolean splitWaitingForAnyNode = false; for (Split split : splits) { if (!split.isRemotelyAccessible()) { - List candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator); + List candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator); if (candidateNodes.isEmpty()) { log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getNodesByHost().keys()); throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query"); } - Node chosenNode = bestNodeSplitCount(candidateNodes.iterator(), minCandidates, maxPendingSplitsPerTask, assignmentStats); + InternalNode chosenNode = bestNodeSplitCount(candidateNodes.iterator(), minCandidates, maxPendingSplitsPerTask, assignmentStats); if (chosenNode != null) { assignment.put(chosenNode, split); assignmentStats.addAssignedSplit(chosenNode); @@ -142,7 +142,7 @@ else if (!splitWaitingForAnyNode) { continue; } - Node chosenNode = null; + InternalNode chosenNode = null; int depth = networkLocationSegmentNames.size(); int chosenDepth = 0; Set locations = new HashSet<>(); @@ -166,7 +166,7 @@ else if (!splitWaitingForAnyNode) { if (filledLocations.contains(location)) { continue; } - Set nodes = nodeMap.getWorkersByNetworkPath().get(location); + Set nodes = nodeMap.getWorkersByNetworkPath().get(location); chosenNode = bestNodeSplitCount(new ResettableRandomizedIterator<>(nodes), minCandidates, calculateMaxPendingSplits(i, depth), assignmentStats); if (chosenNode != null) { chosenDepth = i; @@ -224,14 +224,14 @@ public SplitPlacementResult computeAssignments(Set splits, List candidates, int minCandidatesWhenFull, int maxPendingSplitsPerTask, NodeAssignmentStats assignmentStats) + private InternalNode bestNodeSplitCount(Iterator candidates, int minCandidatesWhenFull, int maxPendingSplitsPerTask, NodeAssignmentStats assignmentStats) { - Node bestQueueNotFull = null; + InternalNode bestQueueNotFull = null; int min = Integer.MAX_VALUE; int fullCandidatesConsidered = 0; while (candidates.hasNext() && (fullCandidatesConsidered < minCandidatesWhenFull || bestQueueNotFull == null)) { - Node node = candidates.next(); + InternalNode node = candidates.next(); if (assignmentStats.getTotalSplitCount(node) < maxSplitsPerNode) { return node; } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicBucketNodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicBucketNodeMap.java index 4bfb13ca33bb4..69113959c5bd2 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicBucketNodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicBucketNodeMap.java @@ -14,8 +14,8 @@ package com.facebook.presto.execution.scheduler.group; import com.facebook.presto.execution.scheduler.BucketNodeMap; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; @@ -30,7 +30,7 @@ public class DynamicBucketNodeMap extends BucketNodeMap { private final int bucketCount; - private final Int2ObjectMap bucketToNode = new Int2ObjectOpenHashMap<>(); + private final Int2ObjectMap bucketToNode = new Int2ObjectOpenHashMap<>(); public DynamicBucketNodeMap(ToIntFunction splitToBucket, int bucketCount) { @@ -40,7 +40,7 @@ public DynamicBucketNodeMap(ToIntFunction splitToBucket, int bucketCount) } @Override - public Optional getAssignedNode(int bucketedId) + public Optional getAssignedNode(int bucketedId) { return Optional.ofNullable(bucketToNode.get(bucketedId)); } @@ -52,7 +52,7 @@ public int getBucketCount() } @Override - public void assignBucketToNode(int bucketedId, Node node) + public void assignBucketToNode(int bucketedId, InternalNode node) { checkArgument(bucketedId >= 0 && bucketedId < bucketCount); requireNonNull(node, "node is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicLifespanScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicLifespanScheduler.java index bc54f33c51b2c..4b51d34c7260b 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicLifespanScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicLifespanScheduler.java @@ -16,7 +16,7 @@ import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.scheduler.BucketNodeMap; import com.facebook.presto.execution.scheduler.SourceScheduler; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.SettableFuture; @@ -43,7 +43,7 @@ public class DynamicLifespanScheduler implements LifespanScheduler { private final BucketNodeMap bucketNodeMap; - private final List allNodes; + private final List allNodes; private final List partitionHandles; private final OptionalInt concurrentLifespansPerTask; @@ -61,7 +61,7 @@ public class DynamicLifespanScheduler @GuardedBy("this") private int totalLifespanExecutionFinished; - public DynamicLifespanScheduler(BucketNodeMap bucketNodeMap, List allNodes, List partitionHandles, OptionalInt concurrentLifespansPerTask) + public DynamicLifespanScheduler(BucketNodeMap bucketNodeMap, List allNodes, List partitionHandles, OptionalInt concurrentLifespansPerTask) { this.bucketNodeMap = requireNonNull(bucketNodeMap, "bucketNodeMap is null"); this.allNodes = requireNonNull(allNodes, "allNodes is null"); @@ -136,7 +136,7 @@ public SettableFuture schedule(SourceScheduler scheduler) } int driverGroupId = driverGroups.nextInt(); - Node nodeForCompletedDriverGroup = bucketNodeMap.getAssignedNode(driverGroup.getId()).orElseThrow(IllegalStateException::new); + InternalNode nodeForCompletedDriverGroup = bucketNodeMap.getAssignedNode(driverGroup.getId()).orElseThrow(IllegalStateException::new); bucketNodeMap.assignBucketToNode(driverGroupId, nodeForCompletedDriverGroup); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/FixedLifespanScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/FixedLifespanScheduler.java index bb6dd83e72744..d80a9f79f7aec 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/FixedLifespanScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/FixedLifespanScheduler.java @@ -16,7 +16,7 @@ import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.scheduler.BucketNodeMap; import com.facebook.presto.execution.scheduler.SourceScheduler; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.SettableFuture; @@ -47,8 +47,8 @@ public class FixedLifespanScheduler implements LifespanScheduler { - private final Int2ObjectMap driverGroupToNodeMap; - private final Map nodeToDriverGroupsMap; + private final Int2ObjectMap driverGroupToNodeMap; + private final Map nodeToDriverGroupsMap; private final List partitionHandles; private final OptionalInt concurrentLifespansPerTask; @@ -64,10 +64,10 @@ public FixedLifespanScheduler(BucketNodeMap bucketNodeMap, List nodeToDriverGroupMap = new HashMap<>(); - Int2ObjectMap driverGroupToNodeMap = new Int2ObjectOpenHashMap<>(); + Map nodeToDriverGroupMap = new HashMap<>(); + Int2ObjectMap driverGroupToNodeMap = new Int2ObjectOpenHashMap<>(); for (int bucket = 0; bucket < bucketNodeMap.getBucketCount(); bucket++) { - Node node = bucketNodeMap.getAssignedNode(bucket).get(); + InternalNode node = bucketNodeMap.getAssignedNode(bucket).get(); nodeToDriverGroupMap.computeIfAbsent(node, key -> new IntArrayList()).add(bucket); driverGroupToNodeMap.put(bucket, node); } @@ -87,7 +87,7 @@ public void scheduleInitial(SourceScheduler scheduler) checkState(!initialScheduled); initialScheduled = true; - for (Map.Entry entry : nodeToDriverGroupsMap.entrySet()) { + for (Map.Entry entry : nodeToDriverGroupsMap.entrySet()) { IntListIterator driverGroupsIterator = entry.getValue(); int driverGroupsScheduled = 0; while (driverGroupsIterator.hasNext()) { diff --git a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java index d0600b34778b2..9ff89fa220a69 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java @@ -18,13 +18,13 @@ import com.facebook.presto.execution.QueryIdGenerator; import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; import com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.server.ServerConfig; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.memory.ClusterMemoryPoolManager; @@ -478,14 +478,14 @@ private synchronized boolean allAssignmentsHavePropagated(Iterable builder = ImmutableSet.builder(); - Set aliveNodes = builder + ImmutableSet.Builder builder = ImmutableSet.builder(); + Set aliveNodes = builder .addAll(nodeManager.getNodes(ACTIVE)) .addAll(nodeManager.getNodes(SHUTTING_DOWN)) .build(); ImmutableSet aliveNodeIds = aliveNodes.stream() - .map(Node::getNodeIdentifier) + .map(InternalNode::getNodeIdentifier) .collect(toImmutableSet()); // Remove nodes that don't exist anymore @@ -494,7 +494,7 @@ private synchronized void updateNodes(MemoryPoolAssignmentsRequest assignments) nodes.keySet().removeAll(deadNodes); // Add new nodes - for (Node node : aliveNodes) { + for (InternalNode node : aliveNodes) { if (!nodes.containsKey(node.getNodeIdentifier())) { nodes.put( node.getNodeIdentifier(), @@ -551,7 +551,7 @@ public synchronized Map> getWorkerMemoryInfo() Map> memoryInfo = new HashMap<>(); for (Entry entry : nodes.entrySet()) { // workerId is of the form "node_identifier [node_host]" - String workerId = entry.getKey() + " [" + entry.getValue().getNode().getHostAndPort().getHostText() + "]"; + String workerId = entry.getKey() + " [" + entry.getValue().getNode().getHost() + "]"; memoryInfo.put(workerId, entry.getValue().getInfo()); } return memoryInfo; diff --git a/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java b/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java index 7daf1beebccbf..df98ce5d8a31e 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.memory; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.server.smile.BaseResponse; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; -import com.facebook.presto.spi.Node; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import io.airlift.http.client.HttpClient; @@ -54,7 +54,7 @@ public class RemoteNodeMemory { private static final Logger log = Logger.get(RemoteNodeMemory.class); - private final Node node; + private final InternalNode node; private final HttpClient httpClient; private final URI memoryInfoUri; private final Codec memoryInfoCodec; @@ -67,7 +67,7 @@ public class RemoteNodeMemory private final boolean isBinaryTransportEnabled; public RemoteNodeMemory( - Node node, + InternalNode node, HttpClient httpClient, Codec memoryInfoCodec, Codec assignmentsRequestCodec, @@ -92,7 +92,7 @@ public Optional getInfo() return memoryInfo.get(); } - public Node getNode() + public InternalNode getNode() { return node; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/AllNodes.java b/presto-main/src/main/java/com/facebook/presto/metadata/AllNodes.java index 4a449ccf35279..d0f9240bbcc01 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/AllNodes.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/AllNodes.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.metadata; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableSet; import java.util.Objects; @@ -23,12 +22,12 @@ public class AllNodes { - private final Set activeNodes; - private final Set inactiveNodes; - private final Set shuttingDownNodes; - private final Set activeCoordinators; + private final Set activeNodes; + private final Set inactiveNodes; + private final Set shuttingDownNodes; + private final Set activeCoordinators; - public AllNodes(Set activeNodes, Set inactiveNodes, Set shuttingDownNodes, Set activeCoordinators) + public AllNodes(Set activeNodes, Set inactiveNodes, Set shuttingDownNodes, Set activeCoordinators) { this.activeNodes = ImmutableSet.copyOf(requireNonNull(activeNodes, "activeNodes is null")); this.inactiveNodes = ImmutableSet.copyOf(requireNonNull(inactiveNodes, "inactiveNodes is null")); @@ -36,22 +35,22 @@ public AllNodes(Set activeNodes, Set inactiveNodes, Set shutti this.activeCoordinators = ImmutableSet.copyOf(requireNonNull(activeCoordinators, "activeCoordinators is null")); } - public Set getActiveNodes() + public Set getActiveNodes() { return activeNodes; } - public Set getInactiveNodes() + public Set getInactiveNodes() { return inactiveNodes; } - public Set getShuttingDownNodes() + public Set getShuttingDownNodes() { return shuttingDownNodes; } - public Set getActiveCoordinators() + public Set getActiveCoordinators() { return activeCoordinators; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index a2cbe9154c5bc..7e1b60f959655 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -18,7 +18,6 @@ import com.facebook.presto.connector.system.GlobalSystemConnector; import com.facebook.presto.failureDetector.FailureDetector; import com.facebook.presto.server.InternalCommunicationConfig; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -81,16 +80,16 @@ public final class DiscoveryNodeManager private final ScheduledExecutorService nodeStateUpdateExecutor; private final ExecutorService nodeStateEventExecutor; private final boolean httpsRequired; - private final PrestoNode currentNode; + private final InternalNode currentNode; @GuardedBy("this") - private SetMultimap activeNodesByConnectorId; + private SetMultimap activeNodesByConnectorId; @GuardedBy("this") private AllNodes allNodes; @GuardedBy("this") - private Set coordinators; + private Set coordinators; @GuardedBy("this") private final List> listeners = new ArrayList<>(); @@ -121,13 +120,13 @@ public DiscoveryNodeManager( refreshNodesInternal(); } - private static PrestoNode findCurrentNode(List allServices, String currentNodeId, NodeVersion expectedNodeVersion, boolean httpsRequired) + private static InternalNode findCurrentNode(List allServices, String currentNodeId, NodeVersion expectedNodeVersion, boolean httpsRequired) { for (ServiceDescriptor service : allServices) { URI uri = getHttpUri(service, httpsRequired); NodeVersion nodeVersion = getNodeVersion(service); if (uri != null && nodeVersion != null) { - PrestoNode node = new PrestoNode(service.getNodeId(), uri, nodeVersion, isCoordinator(service)); + InternalNode node = new InternalNode(service.getNodeId(), uri, nodeVersion, isCoordinator(service)); if (node.getNodeIdentifier().equals(currentNodeId)) { checkState( @@ -159,13 +158,13 @@ public void startPollingNodeStates() private void pollWorkers() { AllNodes allNodes = getAllNodes(); - Set aliveNodes = ImmutableSet.builder() + Set aliveNodes = ImmutableSet.builder() .addAll(allNodes.getActiveNodes()) .addAll(allNodes.getShuttingDownNodes()) .build(); ImmutableSet aliveNodeIds = aliveNodes.stream() - .map(Node::getNodeIdentifier) + .map(InternalNode::getNodeIdentifier) .collect(toImmutableSet()); // Remove nodes that don't exist anymore @@ -174,9 +173,9 @@ private void pollWorkers() nodeStates.keySet().removeAll(deadNodes); // Add new nodes - for (Node node : aliveNodes) { + for (InternalNode node : aliveNodes) { nodeStates.putIfAbsent(node.getNodeIdentifier(), - new RemoteNodeState(httpClient, uriBuilderFrom(node.getHttpUri()).appendPath("/v1/info/state").build())); + new RemoteNodeState(httpClient, uriBuilderFrom(node.getInternalUri()).appendPath("/v1/info/state").build())); } // Schedule refresh @@ -206,18 +205,18 @@ private synchronized void refreshNodesInternal() .filter(service -> !failureDetector.getFailed().contains(service)) .collect(toImmutableSet()); - ImmutableSet.Builder activeNodesBuilder = ImmutableSet.builder(); - ImmutableSet.Builder inactiveNodesBuilder = ImmutableSet.builder(); - ImmutableSet.Builder shuttingDownNodesBuilder = ImmutableSet.builder(); - ImmutableSet.Builder coordinatorsBuilder = ImmutableSet.builder(); - ImmutableSetMultimap.Builder byConnectorIdBuilder = ImmutableSetMultimap.builder(); + ImmutableSet.Builder activeNodesBuilder = ImmutableSet.builder(); + ImmutableSet.Builder inactiveNodesBuilder = ImmutableSet.builder(); + ImmutableSet.Builder shuttingDownNodesBuilder = ImmutableSet.builder(); + ImmutableSet.Builder coordinatorsBuilder = ImmutableSet.builder(); + ImmutableSetMultimap.Builder byConnectorIdBuilder = ImmutableSetMultimap.builder(); for (ServiceDescriptor service : services) { URI uri = getHttpUri(service, httpsRequired); NodeVersion nodeVersion = getNodeVersion(service); boolean coordinator = isCoordinator(service); if (uri != null && nodeVersion != null) { - PrestoNode node = new PrestoNode(service.getNodeId(), uri, nodeVersion, coordinator); + InternalNode node = new InternalNode(service.getNodeId(), uri, nodeVersion, coordinator); NodeState nodeState = getNodeState(node); switch (nodeState) { @@ -253,9 +252,9 @@ private synchronized void refreshNodesInternal() if (allNodes != null) { // log node that are no longer active (but not shutting down) - SetView missingNodes = difference(allNodes.getActiveNodes(), Sets.union(activeNodesBuilder.build(), shuttingDownNodesBuilder.build())); - for (Node missingNode : missingNodes) { - log.info("Previously active node is missing: %s (last seen at %s)", missingNode.getNodeIdentifier(), missingNode.getHostAndPort()); + SetView missingNodes = difference(allNodes.getActiveNodes(), Sets.union(activeNodesBuilder.build(), shuttingDownNodesBuilder.build())); + for (InternalNode missingNode : missingNodes) { + log.info("Previously active node is missing: %s (last seen at %s)", missingNode.getNodeIdentifier(), missingNode.getHost()); } } @@ -275,7 +274,7 @@ private synchronized void refreshNodesInternal() } } - private NodeState getNodeState(PrestoNode node) + private NodeState getNodeState(InternalNode node) { if (expectedNodeVersion.equals(node.getNodeVersion())) { if (isNodeShuttingDown(node.getNodeIdentifier())) { @@ -323,7 +322,7 @@ public int getShuttingDownNodeCount() } @Override - public Set getNodes(NodeState state) + public Set getNodes(NodeState state) { switch (state) { case ACTIVE: @@ -338,19 +337,19 @@ public Set getNodes(NodeState state) } @Override - public synchronized Set getActiveConnectorNodes(ConnectorId connectorId) + public synchronized Set getActiveConnectorNodes(ConnectorId connectorId) { return activeNodesByConnectorId.get(connectorId); } @Override - public Node getCurrentNode() + public InternalNode getCurrentNode() { return currentNode; } @Override - public synchronized Set getCoordinators() + public synchronized Set getCoordinators() { return coordinators; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleJacksonModule.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleJacksonModule.java new file mode 100644 index 0000000000000..f8a8a90053508 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleJacksonModule.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.FunctionHandle; + +import javax.inject.Inject; + +public class FunctionHandleJacksonModule + extends AbstractTypedJacksonModule +{ + @Inject + public FunctionHandleJacksonModule(HandleResolver handleResolver) + { + super(FunctionHandle.class, handleResolver::getId, handleResolver::getFunctionHandleClass); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java new file mode 100644 index 0000000000000..b8ed747087447 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.FunctionHandle; + +public interface FunctionHandleResolver +{ + Class getFunctionHandleClass(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java index 4030a5a919063..d7018ce826564 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java @@ -19,6 +19,8 @@ import com.facebook.presto.operator.window.WindowFunctionSupplier; import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; @@ -33,6 +35,7 @@ @ThreadSafe public class FunctionManager + implements FunctionMetadataManager { private final StaticFunctionNamespace staticFunctionNamespace; private final FunctionInvokerProvider functionInvokerProvider; @@ -61,21 +64,6 @@ public List listFunctions() return staticFunctionNamespace.listFunctions(); } - public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) - { - return staticFunctionNamespace.getFunctionMetadata(functionHandle); - } - - /** - * Lookup up a function with a fully qualified name and fully bound types. - * - * @throws PrestoException if function could not be found - */ - public FunctionHandle lookupFunction(QualifiedName name, List parameterTypes) - { - return staticFunctionNamespace.lookupFunction(name, parameterTypes); - } - /** * Resolves a function using the SQL path, and implicit type coercions. * @@ -91,6 +79,12 @@ public FunctionHandle resolveFunction(Session session, QualifiedName name, List< return staticFunctionNamespace.resolveFunction(name, parameterTypes); } + @Override + public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) + { + return staticFunctionNamespace.getFunctionMetadata(functionHandle); + } + public WindowFunctionSupplier getWindowFunctionImplementation(FunctionHandle functionHandle) { return staticFunctionNamespace.getWindowFunctionImplementation(functionHandle); @@ -116,6 +110,17 @@ public FunctionHandle resolveOperator(OperatorType operatorType, List parameterTypes) + { + return staticFunctionNamespace.lookupFunction(QualifiedName.of(name), parameterTypes); + } + public FunctionHandle lookupCast(CastType castType, TypeSignature fromType, TypeSignature toType) { return staticFunctionNamespace.lookupCast(castType, fromType, toType); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java new file mode 100644 index 0000000000000..20bf0729d1909 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.sql.analyzer.TypeSignatureProvider; +import com.facebook.presto.sql.tree.QualifiedName; + +import java.util.List; + +public interface FunctionNamespace +{ + void addFunctions(List functions); + + List listFunctions(); + + FunctionHandle resolveFunction(QualifiedName name, List parameterTypes); + + FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle); +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java new file mode 100644 index 0000000000000..05b2dd6f1ba67 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +public interface FunctionNamespaceFactory +{ + String getName(); + + FunctionHandleResolver getHandleResolver(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java b/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java index bd70c286b33a9..62238515de1ec 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java @@ -35,6 +35,7 @@ public void configure(Binder binder) jsonBinder(binder).addModuleBinding().to(IndexHandleJacksonModule.class); jsonBinder(binder).addModuleBinding().to(TransactionHandleJacksonModule.class); jsonBinder(binder).addModuleBinding().to(PartitioningHandleJacksonModule.class); + jsonBinder(binder).addModuleBinding().to(FunctionHandleJacksonModule.class); binder.bind(HandleResolver.class).in(Scopes.SINGLETON); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java index 4a45072216238..cea0264385aca 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.split.EmptySplitHandleResolver; import javax.inject.Inject; @@ -45,6 +46,7 @@ public class HandleResolver { private final ConcurrentMap handleResolvers = new ConcurrentHashMap<>(); + private final ConcurrentMap functionHandleResolvers = new ConcurrentHashMap<>(); @Inject public HandleResolver() @@ -53,6 +55,8 @@ public HandleResolver() handleResolvers.put("$system", new MaterializedHandleResolver(new SystemHandleResolver())); handleResolvers.put("$info_schema", new MaterializedHandleResolver(new InformationSchemaHandleResolver())); handleResolvers.put("$empty", new MaterializedHandleResolver(new EmptySplitHandleResolver())); + + functionHandleResolvers.put("$static", new MaterializedFunctionHandleResolver(new StaticFunctionNamespaceHandleResolver())); } public void addConnectorName(String name, ConnectorHandleResolver resolver) @@ -64,6 +68,14 @@ public void addConnectorName(String name, ConnectorHandleResolver resolver) "Connector '%s' is already assigned to resolver: %s", name, existingResolver); } + public void addFunctionNamepsace(String name, FunctionHandleResolver resolver) + { + requireNonNull(name, "name is null"); + requireNonNull(resolver, "resolver is null"); + MaterializedFunctionHandleResolver existingResolver = functionHandleResolvers.putIfAbsent(name, new MaterializedFunctionHandleResolver(resolver)); + checkState(existingResolver == null || existingResolver.equals(resolver), "Name %s is already assigned to function resolver: %s", name, existingResolver); + } + public String getId(ConnectorTableHandle tableHandle) { return getId(tableHandle, MaterializedHandleResolver::getTableHandleClass); @@ -109,6 +121,11 @@ public String getId(ConnectorTransactionHandle transactionHandle) return getId(transactionHandle, MaterializedHandleResolver::getTransactionHandleClass); } + public String getId(FunctionHandle functionHandle) + { + return getFunctionNamespaceId(functionHandle, MaterializedFunctionHandleResolver::getFunctionHandleClass); + } + public Class getTableHandleClass(String id) { return resolverFor(id).getTableHandleClass().orElseThrow(() -> new IllegalArgumentException("No resolver for " + id)); @@ -154,6 +171,11 @@ public Class getTransactionHandleClass(Str return resolverFor(id).getTransactionHandleClass().orElseThrow(() -> new IllegalArgumentException("No resolver for " + id)); } + public Class getFunctionHandleClass(String id) + { + return resolverForFunctionNamespace(id).getFunctionHandleClass().orElseThrow(() -> new IllegalArgumentException("No resolver for " + id)); + } + private MaterializedHandleResolver resolverFor(String id) { MaterializedHandleResolver resolver = handleResolvers.get(id); @@ -161,6 +183,13 @@ private MaterializedHandleResolver resolverFor(String id) return resolver; } + private MaterializedFunctionHandleResolver resolverForFunctionNamespace(String id) + { + MaterializedFunctionHandleResolver resolver = functionHandleResolvers.get(id); + checkArgument(resolver != null, "No handle resolver for function namespace: %s", id); + return resolver; + } + private String getId(T handle, Function>> getter) { for (Entry entry : handleResolvers.entrySet()) { @@ -175,6 +204,20 @@ private String getId(T handle, Function String getFunctionNamespaceId(T handle, Function>> getter) + { + for (Entry entry : functionHandleResolvers.entrySet()) { + try { + if (getter.apply(entry.getValue()).map(clazz -> clazz.isInstance(handle)).orElse(false)) { + return entry.getKey(); + } + } + catch (UnsupportedOperationException ignored) { + } + } + throw new IllegalArgumentException("No function namespace for handle: " + handle); + } + private static class MaterializedHandleResolver { private final Optional> tableHandle; @@ -282,4 +325,48 @@ public int hashCode() return Objects.hash(tableHandle, layoutHandle, columnHandle, split, indexHandle, outputTableHandle, insertTableHandle, partitioningHandle, transactionHandle); } } + + private static class MaterializedFunctionHandleResolver + { + private final Optional> functionHandle; + + public MaterializedFunctionHandleResolver(FunctionHandleResolver resolver) + { + functionHandle = getHandleClass(resolver::getFunctionHandleClass); + } + + private static Optional> getHandleClass(Supplier> callable) + { + try { + return Optional.of(callable.get()); + } + catch (UnsupportedOperationException e) { + return Optional.empty(); + } + } + + public Optional> getFunctionHandleClass() + { + return functionHandle; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MaterializedFunctionHandleResolver that = (MaterializedFunctionHandleResolver) o; + return Objects.equals(functionHandle, that.functionHandle); + } + + @Override + public int hashCode() + { + return Objects.hash(functionHandle); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java index 8df5edec44637..9a550b207f771 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java @@ -15,7 +15,6 @@ import com.facebook.presto.client.NodeVersion; import com.facebook.presto.connector.ConnectorId; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; @@ -37,8 +36,8 @@ public class InMemoryNodeManager implements InternalNodeManager { - private final Node localNode; - private final SetMultimap remoteNodes = Multimaps.synchronizedSetMultimap(HashMultimap.create()); + private final InternalNode localNode; + private final SetMultimap remoteNodes = Multimaps.synchronizedSetMultimap(HashMultimap.create()); @GuardedBy("this") private final List> listeners = new ArrayList<>(); @@ -51,7 +50,7 @@ public InMemoryNodeManager() public InMemoryNodeManager(URI localUri) { - localNode = new PrestoNode("local", localUri, NodeVersion.UNKNOWN, false); + localNode = new InternalNode("local", localUri, NodeVersion.UNKNOWN, false); } public void addCurrentNodeConnector(ConnectorId connectorId) @@ -59,12 +58,12 @@ public void addCurrentNodeConnector(ConnectorId connectorId) addNode(connectorId, localNode); } - public void addNode(ConnectorId connectorId, Node... nodes) + public void addNode(ConnectorId connectorId, InternalNode... nodes) { addNode(connectorId, ImmutableList.copyOf(nodes)); } - public void addNode(ConnectorId connectorId, Iterable nodes) + public void addNode(ConnectorId connectorId, Iterable nodes) { remoteNodes.putAll(connectorId, nodes); @@ -77,7 +76,7 @@ public void addNode(ConnectorId connectorId, Iterable nodes) } @Override - public Set getNodes(NodeState state) + public Set getNodes(NodeState state) { switch (state) { case ACTIVE: @@ -92,7 +91,7 @@ public Set getNodes(NodeState state) } @Override - public Set getActiveConnectorNodes(ConnectorId connectorId) + public Set getActiveConnectorNodes(ConnectorId connectorId) { return ImmutableSet.copyOf(remoteNodes.get(connectorId)); } @@ -100,17 +99,17 @@ public Set getActiveConnectorNodes(ConnectorId connectorId) @Override public AllNodes getAllNodes() { - return new AllNodes(ImmutableSet.builder().add(localNode).addAll(remoteNodes.values()).build(), ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of(localNode)); + return new AllNodes(ImmutableSet.builder().add(localNode).addAll(remoteNodes.values()).build(), ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of(localNode)); } @Override - public Node getCurrentNode() + public InternalNode getCurrentNode() { return localNode; } @Override - public Set getCoordinators() + public Set getCoordinators() { // always use localNode as coordinator return ImmutableSet.of(localNode); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/PrestoNode.java b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java similarity index 81% rename from presto-main/src/main/java/com/facebook/presto/metadata/PrestoNode.java rename to presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java index 2dfcfb86bf71f..b869e4375b7bb 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/PrestoNode.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java @@ -27,19 +27,19 @@ /** * A node is a server in a cluster than can process queries. */ -public class PrestoNode +public class InternalNode implements Node { private final String nodeIdentifier; - private final URI httpUri; + private final URI internalUri; private final NodeVersion nodeVersion; private final boolean coordinator; - public PrestoNode(String nodeIdentifier, URI httpUri, NodeVersion nodeVersion, boolean coordinator) + public InternalNode(String nodeIdentifier, URI internalUri, NodeVersion nodeVersion, boolean coordinator) { nodeIdentifier = emptyToNull(nullToEmpty(nodeIdentifier).trim()); this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null or empty"); - this.httpUri = requireNonNull(httpUri, "httpUri is null"); + this.internalUri = requireNonNull(internalUri, "internalUri is null"); this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null"); this.coordinator = coordinator; } @@ -51,15 +51,27 @@ public String getNodeIdentifier() } @Override + public String getHost() + { + return internalUri.getHost(); + } + + @Override + @Deprecated public URI getHttpUri() { - return httpUri; + return getInternalUri(); + } + + public URI getInternalUri() + { + return internalUri; } @Override public HostAddress getHostAndPort() { - return HostAddress.fromUri(httpUri); + return HostAddress.fromUri(internalUri); } @Override @@ -88,7 +100,7 @@ public boolean equals(Object obj) if ((obj == null) || (getClass() != obj.getClass())) { return false; } - PrestoNode o = (PrestoNode) obj; + InternalNode o = (InternalNode) obj; return nodeIdentifier.equals(o.nodeIdentifier); } @@ -103,7 +115,7 @@ public String toString() { return toStringHelper(this) .add("nodeIdentifier", nodeIdentifier) - .add("httpUri", httpUri) + .add("internalUri", internalUri) .add("nodeVersion", nodeVersion) .toString(); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java index a0d13c6f2754d..a6b63b5ee6a38 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java @@ -14,7 +14,6 @@ package com.facebook.presto.metadata; import com.facebook.presto.connector.ConnectorId; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import java.util.Set; @@ -22,13 +21,13 @@ public interface InternalNodeManager { - Set getNodes(NodeState state); + Set getNodes(NodeState state); - Set getActiveConnectorNodes(ConnectorId connectorId); + Set getActiveConnectorNodes(ConnectorId connectorId); - Node getCurrentNode(); + InternalNode getCurrentNode(); - Set getCoordinators(); + Set getCoordinators(); AllNodes getAllNodes(); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java new file mode 100644 index 0000000000000..ce5e7fa82d3e3 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.CatalogSchemaName; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.Signature; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class StaticFunctionHandle + implements FunctionHandle +{ + private static final CatalogSchemaName STATIC_FUNCTION_NAMESPACE_CATALOG_SCHEMA_NAME = new CatalogSchemaName("static", "system"); + private final Signature signature; + + @JsonCreator + public StaticFunctionHandle(@JsonProperty("signature") Signature signature) + { + this.signature = requireNonNull(signature, "signature is null"); + checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); + } + + @JsonProperty + public Signature getSignature() + { + return signature; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StaticFunctionHandle that = (StaticFunctionHandle) o; + return Objects.equals(signature, that.signature); + } + + @Override + public CatalogSchemaName getCatalogSchemaName() + { + return STATIC_FUNCTION_NAMESPACE_CATALOG_SCHEMA_NAME; + } + + @Override + public int hashCode() + { + return Objects.hash(signature); + } + + @Override + public String toString() + { + return signature.toString(); + } + + private static void checkArgument(boolean condition, String message, Object... args) + { + if (!condition) { + throw new IllegalArgumentException(String.format(message, args)); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java index 334f611ab24bc..582bc81f65ac8 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java @@ -148,6 +148,7 @@ import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; @@ -358,6 +359,7 @@ @ThreadSafe class StaticFunctionNamespace + implements FunctionNamespace { private final TypeManager typeManager; private final LoadingCache specializedFunctionKeyCache; @@ -664,6 +666,7 @@ public StaticFunctionNamespace( addFunctions(builder.getFunctions()); } + @Override public final synchronized void addFunctions(List functions) { for (SqlFunction function : functions) { @@ -674,6 +677,7 @@ public final synchronized void addFunctions(List function this.functions = new FunctionMap(this.functions, functions); } + @Override public List listFunctions() { return functions.list().stream() @@ -686,18 +690,20 @@ public boolean isAggregationFunction(QualifiedName name) return Iterables.any(functions.get(name), function -> function.getSignature().getKind() == AGGREGATE); } + @Override public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) { + checkArgument(functionHandle instanceof StaticFunctionHandle, "Expect StaticFunctionHandle"); + Signature signature = ((StaticFunctionHandle) functionHandle).getSignature(); SpecializedFunctionKey functionKey; try { - functionKey = specializedFunctionKeyCache.getUnchecked(functionHandle.getSignature()); + functionKey = specializedFunctionKeyCache.getUnchecked(signature); } catch (UncheckedExecutionException e) { throwIfInstanceOf(e.getCause(), PrestoException.class); throw e; } SqlFunction function = functionKey.getFunction(); - Signature signature = functionHandle.getSignature(); Optional operatorType = tryGetOperatorType(signature.getName()); if (operatorType.isPresent()) { return new FunctionMetadata( @@ -728,7 +734,7 @@ public FunctionHandle lookupFunction(QualifiedName name, List match = matchFunctionExact(exactCandidates, parameterTypes); if (match.isPresent()) { - return new FunctionHandle(match.get()); + return new StaticFunctionHandle(match.get()); } List genericCandidates = allCandidates.stream() @@ -737,12 +743,13 @@ public FunctionHandle lookupFunction(QualifiedName name, List parameterTypes) { try { @@ -757,7 +764,7 @@ public FunctionHandle resolveFunction(QualifiedName name, List allCandidates = functions.get(name); Optional match = matchFunctionWithCoercion(allCandidates, parameterTypes); if (match.isPresent()) { - return new FunctionHandle(match.get()); + return new StaticFunctionHandle(match.get()); } if (name.getSuffix().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) { @@ -770,7 +777,7 @@ public FunctionHandle resolveFunction(QualifiedName name, List> toTypes(List typeSignatureProviders, TypeManager typeManager) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java new file mode 100644 index 0000000000000..14e4fed325e99 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +public class StaticFunctionNamespaceFactory + implements FunctionNamespaceFactory +{ + @Override + public String getName() + { + return "$static"; + } + + @Override + public FunctionHandleResolver getHandleResolver() + { + return new StaticFunctionNamespaceHandleResolver(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java new file mode 100644 index 0000000000000..19792a7eccaa1 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.FunctionHandle; + +public class StaticFunctionNamespaceHandleResolver + implements FunctionHandleResolver +{ + @Override + public Class getFunctionHandleClass() + { + return StaticFunctionHandle.class; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AccumulatorCompiler.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AccumulatorCompiler.java index ab550944e27ed..dd78da3c43d79 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AccumulatorCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AccumulatorCompiler.java @@ -218,7 +218,7 @@ private static MethodDefinition generateGetIntermediateType(ClassDefinition defi } else { methodDefinition.getBody() - .append(constantType(callSiteBinder, RowType.anonymous(type))) + .append(constantType(callSiteBinder, RowType.withDefaultFieldNames(type))) .retObject(); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/InternalAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/InternalAggregationFunction.java index f2485704a23f2..25e0df6d4030b 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/InternalAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/InternalAggregationFunction.java @@ -102,7 +102,7 @@ public Type getIntermediateType() return getOnlyElement(intermediateType); } else { - return RowType.anonymous(intermediateType); + return RowType.withDefaultFieldNames(intermediateType); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java index ce1516177b4b9..efe277f211597 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java @@ -80,13 +80,13 @@ public InternalAggregationFunction specialize(BoundVariables boundVariables, int Type valueType = boundVariables.getTypeVariable("T"); QuantileDigestType outputType = (QuantileDigestType) typeManager.getParameterizedType(StandardTypes.QDIGEST, ImmutableList.of(TypeSignatureParameter.of(valueType.getTypeSignature()))); - return generateAggregation(valueType, outputType); + return generateAggregation(outputType); } - private static InternalAggregationFunction generateAggregation(Type valueType, QuantileDigestType type) + private static InternalAggregationFunction generateAggregation(QuantileDigestType type) { DynamicClassLoader classLoader = new DynamicClassLoader(MapAggregationFunction.class.getClassLoader()); - QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(valueType); + QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(); Type intermediateType = stateSerializer.getSerializedType(); AggregationMetadata metadata = new AggregationMetadata( diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java index 72d6d7c1a1ea3..b59abd51dfe58 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java @@ -99,7 +99,7 @@ private static InternalAggregationFunction generateAggregation(Type valueType, Q { DynamicClassLoader classLoader = new DynamicClassLoader(QuantileDigestAggregationFunction.class.getClassLoader()); List inputTypes = getInputTypes(valueType, arity); - QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(valueType); + QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(); Type intermediateType = stateSerializer.getSerializedType(); AggregationMetadata metadata = new AggregationMetadata( diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedKeyValueHeap.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedKeyValueHeap.java index 70ccb03fbbf6d..c062835f83baa 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedKeyValueHeap.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedKeyValueHeap.java @@ -56,7 +56,7 @@ public TypedKeyValueHeap(BlockComparator keyComparator, Type keyType, Type value public static Type getSerializedType(Type keyType, Type valueType) { - return RowType.anonymous(ImmutableList.of(BIGINT, new ArrayType(keyType), new ArrayType(valueType))); + return RowType.withDefaultFieldNames(ImmutableList.of(BIGINT, new ArrayType(keyType), new ArrayType(valueType))); } public int getCapacity() diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/KeyAndBlockPositionValueStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/KeyAndBlockPositionValueStateSerializer.java index 22fa53fdaad6f..ef6f9f04f0af1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/KeyAndBlockPositionValueStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/KeyAndBlockPositionValueStateSerializer.java @@ -47,7 +47,7 @@ public abstract class KeyAndBlockPositionValueStateSerializer @@ -27,7 +27,7 @@ public class HyperLogLogStateSerializer @Override public Type getSerializedType() { - return HYPER_LOG_LOG; + return VARBINARY; } @Override @@ -37,13 +37,13 @@ public void serialize(HyperLogLogState state, BlockBuilder out) out.appendNull(); } else { - HYPER_LOG_LOG.writeSlice(out, state.getHyperLogLog().serialize()); + VARBINARY.writeSlice(out, state.getHyperLogLog().serialize()); } } @Override public void deserialize(Block block, int index, HyperLogLogState state) { - state.setHyperLogLog(HyperLogLog.newInstance(HYPER_LOG_LOG.getSlice(block, index))); + state.setHyperLogLog(HyperLogLog.newInstance(VARBINARY.getSlice(block, index))); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/MinMaxNStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/MinMaxNStateSerializer.java index 7973a94cd3935..109953afe39d6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/MinMaxNStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/MinMaxNStateSerializer.java @@ -39,7 +39,7 @@ public MinMaxNStateSerializer(BlockComparator blockComparator, Type elementType) this.blockComparator = blockComparator; this.elementType = elementType; this.arrayType = new ArrayType(elementType); - this.serializedType = RowType.anonymous(ImmutableList.of(BIGINT, arrayType)); + this.serializedType = RowType.withDefaultFieldNames(ImmutableList.of(BIGINT, arrayType)); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java index e51fb4d60ac4e..aa1d33ea14dd1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java @@ -16,24 +16,18 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; -import com.facebook.presto.spi.type.QuantileDigestType; import com.facebook.presto.spi.type.Type; import io.airlift.stats.QuantileDigest; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; + public class QuantileDigestStateSerializer implements AccumulatorStateSerializer { - private final QuantileDigestType type; - - public QuantileDigestStateSerializer(Type elementType) - { - this.type = new QuantileDigestType(elementType); - } - @Override public Type getSerializedType() { - return type; + return VARBINARY; } @Override @@ -43,13 +37,13 @@ public void serialize(QuantileDigestState state, BlockBuilder out) out.appendNull(); } else { - type.writeSlice(out, state.getQuantileDigest().serialize()); + VARBINARY.writeSlice(out, state.getQuantileDigest().serialize()); } } @Override public void deserialize(Block block, int index, QuantileDigestState state) { - state.setQuantileDigest(new QuantileDigest(type.getSlice(block, index))); + state.setQuantileDigest(new QuantileDigest(VARBINARY.getSlice(block, index))); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StateCompiler.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StateCompiler.java index 771f6651cdd02..7febe2a47739c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StateCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StateCompiler.java @@ -186,7 +186,7 @@ private static void generateGetSerializedType(ClassDefinition definition, List 1) { List types = fields.stream().map(StateField::getSqlType).collect(toImmutableList()); - type = RowType.anonymous(types); + type = RowType.withDefaultFieldNames(types); } else if (fields.size() == 1) { type = getOnlyElement(fields).getSqlType(); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/annotations/FunctionImplementationDependency.java b/presto-main/src/main/java/com/facebook/presto/operator/annotations/FunctionImplementationDependency.java index c1b47c16510ec..a6c8332b163e6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/annotations/FunctionImplementationDependency.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/annotations/FunctionImplementationDependency.java @@ -18,7 +18,6 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.InvocationConvention; import com.facebook.presto.spi.type.TypeSignature; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import java.util.List; @@ -45,7 +44,7 @@ public FunctionImplementationDependency(String name, List argumen @Override protected FunctionHandle getFunctionHandle(BoundVariables boundVariables, FunctionManager functionManager) { - return functionManager.lookupFunction(QualifiedName.of(name), fromTypeSignatures(applyBoundVariables(argumentTypes, boundVariables))); + return functionManager.lookupFunction(name, fromTypeSignatures(applyBoundVariables(argumentTypes, boundVariables))); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java index a4031aef877ed..9d046bb2be5b5 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java @@ -16,11 +16,11 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.CastType; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.SqlOperator; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.ArrayGeneratorUtils; diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index a93f2f307db9a..19f8ebe6c7644 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -83,6 +83,8 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.block.BlockEncodingSerde; +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spiller.FileSingleStreamSpillerFactory; @@ -109,11 +111,13 @@ import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; import com.facebook.presto.sql.gen.OrderingCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; +import com.facebook.presto.sql.gen.RowExpressionPredicateCompiler; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.CompilerConfig; import com.facebook.presto.sql.planner.LocalExecutionPlanner; import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.transaction.TransactionManagerConfig; @@ -358,6 +362,10 @@ protected void setup(Binder binder) binder.bind(MetadataManager.class).in(Scopes.SINGLETON); binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON); + // row expression utils + binder.bind(DomainTranslator.class).to(RowExpressionDomainTranslator.class).in(Scopes.SINGLETON); + binder.bind(PredicateCompiler.class).to(RowExpressionPredicateCompiler.class).in(Scopes.SINGLETON); + // type binder.bind(TypeRegistry.class).in(Scopes.SINGLETON); binder.bind(TypeManager.class).to(TypeRegistry.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java b/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java index b8095b3422633..b1a575a37e479 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import io.airlift.http.client.HttpClient; import io.airlift.http.client.Request; @@ -68,14 +68,14 @@ public Response getThreads(@PathParam("nodeId") String nodeId) private Response proxyJsonResponse(String nodeId, String workerPath) { - Set nodes = nodeManager.getNodes(NodeState.ACTIVE); - Node node = nodes.stream() + Set nodes = nodeManager.getNodes(NodeState.ACTIVE); + InternalNode node = nodes.stream() .filter(n -> n.getNodeIdentifier().equals(nodeId)) .findFirst() .orElseThrow(() -> new WebApplicationException(NOT_FOUND)); Request request = prepareGet() - .setUri(uriBuilderFrom(node.getHttpUri()) + .setUri(uriBuilderFrom(node.getInternalUri()) .appendPath(workerPath) .build()) .build(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java index 878fb511508a8..ee961e63c948f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java @@ -16,9 +16,9 @@ import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.StageId; import com.facebook.presto.execution.TaskId; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.server.InternalCommunicationConfig; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; import io.airlift.http.server.HttpServerInfo; @@ -74,21 +74,21 @@ public URI createLocalTaskLocation(TaskId taskId) } @Override - public URI createTaskLocation(Node node, TaskId taskId) + public URI createTaskLocation(InternalNode node, TaskId taskId) { requireNonNull(node, "node is null"); requireNonNull(taskId, "taskId is null"); - return uriBuilderFrom(node.getHttpUri()) + return uriBuilderFrom(node.getInternalUri()) .appendPath("/v1/task") .appendPath(taskId.toString()) .build(); } @Override - public URI createMemoryInfoLocation(Node node) + public URI createMemoryInfoLocation(InternalNode node) { requireNonNull(node, "node is null"); - return uriBuilderFrom(node.getHttpUri()) + return uriBuilderFrom(node.getInternalUri()) .appendPath("/v1/memory").build(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java index d1aae612c7365..c496258a745ed 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java @@ -34,6 +34,7 @@ import com.facebook.presto.server.smile.BaseResponse; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; @@ -45,15 +46,18 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import io.airlift.concurrent.SetThreadName; import io.airlift.http.client.HttpClient; import io.airlift.http.client.HttpUriBuilder; import io.airlift.http.client.Request; import io.airlift.http.client.ResponseHandler; +import io.airlift.http.client.StatusResponseHandler.StatusResponse; import io.airlift.log.Logger; import io.airlift.units.Duration; import org.joda.time.DateTime; +import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import java.net.URI; @@ -85,16 +89,21 @@ import static com.facebook.presto.server.smile.AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler; import static com.facebook.presto.server.smile.FullSmileResponseHandler.createFullSmileResponseHandler; import static com.facebook.presto.server.smile.JsonCodecWrapper.unwrapJsonCodec; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.util.Failures.toFailure; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.util.concurrent.Futures.addCallback; import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static io.airlift.http.client.HttpStatus.OK; import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static io.airlift.http.client.Request.Builder.prepareDelete; import static io.airlift.http.client.Request.Builder.preparePost; import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -106,6 +115,7 @@ public final class HttpRemoteTask private static final Logger log = Logger.get(HttpRemoteTask.class); private final TaskId taskId; + private final URI taskLocation; private final Session session; private final String nodeId; @@ -114,6 +124,7 @@ public final class HttpRemoteTask private final AtomicLong nextSplitId = new AtomicLong(); + private final Duration maxErrorDuration; private final RemoteTaskStats stats; private final TaskInfoFetcher taskInfoFetcher; private final ContinuousTaskStatusFetcher taskStatusFetcher; @@ -197,10 +208,12 @@ public HttpRemoteTask(Session session, requireNonNull(taskInfoCodec, "taskInfoCodec is null"); requireNonNull(taskUpdateRequestCodec, "taskUpdateRequestCodec is null"); requireNonNull(partitionedSplitCountTracker, "partitionedSplitCountTracker is null"); + requireNonNull(maxErrorDuration, "maxErrorDuration is null"); requireNonNull(stats, "stats is null"); try (SetThreadName ignored = new SetThreadName("HttpRemoteTask-%s", taskId)) { this.taskId = taskId; + this.taskLocation = location; this.session = session; this.nodeId = nodeId; this.planFragment = planFragment; @@ -214,6 +227,7 @@ public HttpRemoteTask(Session session, this.taskUpdateRequestCodec = taskUpdateRequestCodec; this.updateErrorTracker = new RequestErrorTracker(taskId, location, maxErrorDuration, errorScheduledExecutor, "updating task"); this.partitionedSplitCountTracker = requireNonNull(partitionedSplitCountTracker, "partitionedSplitCountTracker is null"); + this.maxErrorDuration = maxErrorDuration; this.stats = stats; this.isBinaryTransportEnabled = isBinaryTransportEnabled; @@ -382,6 +396,77 @@ public synchronized void setOutputBuffers(OutputBuffers newOutputBuffers) } } + @Override + public ListenableFuture removeRemoteSource(TaskId remoteSourceTaskId) + { + URI remoteSourceUri = uriBuilderFrom(taskLocation) + .appendPath("remote-source") + .appendPath(remoteSourceTaskId.toString()) + .build(); + + Request request = prepareDelete() + .setUri(remoteSourceUri) + .build(); + RequestErrorTracker errorTracker = new RequestErrorTracker( + taskId, + remoteSourceUri, + maxErrorDuration, + errorScheduledExecutor, + "Remove exchange remote source"); + + SettableFuture future = SettableFuture.create(); + doRemoveRemoteSource(errorTracker, request, future); + return future; + } + + /// This method may call itself recursively when retrying for failures + private void doRemoveRemoteSource(RequestErrorTracker errorTracker, Request request, SettableFuture future) + { + errorTracker.startRequest(); + + FutureCallback callback = new FutureCallback() { + @Override + public void onSuccess(@Nullable StatusResponse response) + { + if (response == null) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Request failed with null response"); + } + if (response.getStatusCode() != OK.code()) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Request failed with HTTP status " + response.getStatusCode()); + } + future.set(null); + } + + @Override + public void onFailure(Throwable failedReason) + { + if (failedReason instanceof RejectedExecutionException && httpClient.isClosed()) { + log.error("Unable to destroy exchange source at %s. HTTP client is closed", request.getUri()); + future.setException(failedReason); + return; + } + // record failure + try { + errorTracker.requestFailed(failedReason); + } + catch (PrestoException e) { + future.setException(e); + return; + } + // if throttled due to error, asynchronously wait for timeout and try again + ListenableFuture errorRateLimit = errorTracker.acquireRequestPermit(); + if (errorRateLimit.isDone()) { + doRemoveRemoteSource(errorTracker, request, future); + } + else { + errorRateLimit.addListener(() -> doRemoveRemoteSource(errorTracker, request, future), errorScheduledExecutor); + } + } + }; + + addCallback(httpClient.executeAsync(request, createStatusResponseHandler()), callback, directExecutor()); + } + @Override public int getPartitionedSplitCount() { diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java index 23cd8bbbf8329..4f334781243a7 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java @@ -24,52 +24,36 @@ import com.facebook.presto.execution.TaskManagerConfig; import com.facebook.presto.execution.TaskStatus; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.ForScheduler; import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.server.TaskUpdateRequest; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; -import com.facebook.presto.spi.Node; -import com.facebook.presto.spi.PrestoException; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.collect.Multimap; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; import io.airlift.concurrent.BoundedExecutor; import io.airlift.concurrent.ThreadPoolExecutorMBean; import io.airlift.http.client.HttpClient; -import io.airlift.http.client.Request; -import io.airlift.http.client.StatusResponseHandler.StatusResponse; import io.airlift.json.JsonCodec; import io.airlift.log.Logger; import io.airlift.units.Duration; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; -import javax.annotation.Nullable; import javax.annotation.PreDestroy; import javax.inject.Inject; -import java.net.URI; import java.util.OptionalInt; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import static com.facebook.presto.server.smile.JsonCodecWrapper.wrapJsonCodec; -import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static com.google.common.util.concurrent.Futures.addCallback; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.http.client.HttpStatus.OK; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; @@ -153,7 +137,7 @@ public void stop() @Override public RemoteTask createRemoteTask(Session session, TaskId taskId, - Node node, + InternalNode node, PlanFragment fragment, Multimap initialSplits, OptionalInt totalPartitions, @@ -184,74 +168,4 @@ public RemoteTask createRemoteTask(Session session, stats, isBinaryTransportEnabled); } - - @Override - public ListenableFuture removeRemoteSource(TaskId taskId, TaskId remoteSourceTaskId) - { - URI remoteSourceUri = uriBuilderFrom(locationFactory.createLocalTaskLocation(taskId)) - .appendPath("remote-source") - .appendPath(remoteSourceTaskId.toString()) - .build(); - - Request request = prepareDelete() - .setUri(remoteSourceUri) - .build(); - RequestErrorTracker errorTracker = new RequestErrorTracker( - taskId, - remoteSourceUri, - maxErrorDuration, - errorScheduledExecutor, - "Remove exchange remote source"); - - SettableFuture future = SettableFuture.create(); - removeRemoteSourceHelper(errorTracker, request, future); - return future; - } - - private void removeRemoteSourceHelper(RequestErrorTracker errorTracker, Request request, SettableFuture future) - { - errorTracker.startRequest(); - - FutureCallback callback = new FutureCallback() { - @Override - public void onSuccess(@Nullable StatusResponse response) - { - if (response == null) { - throw new PrestoException(GENERIC_INTERNAL_ERROR, "Request failed with null response"); - } - if (response.getStatusCode() != OK.code()) { - throw new PrestoException(GENERIC_INTERNAL_ERROR, "Request failed with HTTP status " + response.getStatusCode()); - } - future.set(null); - } - - @Override - public void onFailure(Throwable failedReason) - { - if (failedReason instanceof RejectedExecutionException && httpClient.isClosed()) { - LOG.error("Unable to destroy exchange source at %s. HTTP client is closed", request.getUri()); - future.setException(failedReason); - return; - } - // record failure - try { - errorTracker.requestFailed(failedReason); - } - catch (PrestoException e) { - future.setException(e); - return; - } - // if throttled due to error, asynchronously wait for timeout and try again - ListenableFuture errorRateLimit = errorTracker.acquireRequestPermit(); - if (errorRateLimit.isDone()) { - removeRemoteSourceHelper(errorTracker, request, future); - } - else { - errorRateLimit.addListener(() -> removeRemoteSourceHelper(errorTracker, request, future), errorScheduledExecutor); - } - } - }; - - addCallback(httpClient.executeAsync(request, createStatusResponseHandler()), callback, directExecutor()); - } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index 13a0bab62749b..66d68f7f5e42f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -27,6 +27,7 @@ import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.metadata.AllNodes; import com.facebook.presto.metadata.CatalogManager; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.security.AccessControl; @@ -37,7 +38,6 @@ import com.facebook.presto.server.ShutdownAction; import com.facebook.presto.server.security.ServerSecurityModule; import com.facebook.presto.server.smile.SmileModule; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.QueryId; import com.facebook.presto.split.PageSourceManager; @@ -493,7 +493,7 @@ public final AllNodes refreshNodes() return nodeManager.getAllNodes(); } - public Set getActiveNodesWithConnector(ConnectorId connectorId) + public Set getActiveNodesWithConnector(ConnectorId connectorId) { return nodeManager.getActiveConnectorNodes(connectorId); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java index a290bcfdc89cd..231cc4e75107f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java @@ -17,7 +17,6 @@ import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.OperatorNotFoundException; import com.facebook.presto.metadata.QualifiedObjectName; @@ -26,6 +25,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalParseResult; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticExceptions.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticExceptions.java index ac322a1c600e3..e2f158f26b19b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticExceptions.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticExceptions.java @@ -15,11 +15,13 @@ import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Node; +import com.facebook.presto.sql.tree.NodeLocation; import com.facebook.presto.sql.tree.QualifiedName; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.AMBIGUOUS_ATTRIBUTE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_ATTRIBUTE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; +import static java.lang.String.format; public final class SemanticExceptions { @@ -48,4 +50,13 @@ public static SemanticException notSupportedException(Node node, String notSuppo { throw new SemanticException(NOT_SUPPORTED, node, notSupportedFeatureDescription + " is not supported"); } + + public static String subQueryNotSupportedError(Node node, String notSupportedFeatureDescription) + { + if (node.getLocation().isPresent()) { + NodeLocation nodeLocation = node.getLocation().get(); + return format("line %s:%s: %s", nodeLocation.getLineNumber(), nodeLocation.getColumnNumber(), notSupportedFeatureDescription + " is not supported"); + } + return notSupportedFeatureDescription + " is not supported"; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/WindowFunctionValidator.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/WindowFunctionValidator.java index 4963192de49ee..f5597d8d30e8f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/WindowFunctionValidator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/WindowFunctionValidator.java @@ -14,7 +14,7 @@ package com.facebook.presto.sql.analyzer; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.sql.tree.DefaultExpressionTraversalVisitor; import com.facebook.presto.sql.tree.FunctionCall; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java index 975e610bee1ff..f63039da41ea5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java @@ -15,9 +15,9 @@ import com.facebook.presto.metadata.CastType; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeSignature; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionPredicateCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionPredicateCompiler.java new file mode 100644 index 0000000000000..8d9a1f15feae3 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionPredicateCompiler.java @@ -0,0 +1,301 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.gen; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.operator.project.PageFieldsToInputParametersRewriter; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.Predicate; +import com.facebook.presto.spi.relation.PredicateCompiler; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.sql.relational.DeterminismEvaluator; +import com.facebook.presto.sql.relational.Expressions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.airlift.bytecode.BytecodeBlock; +import io.airlift.bytecode.BytecodeNode; +import io.airlift.bytecode.ClassDefinition; +import io.airlift.bytecode.MethodDefinition; +import io.airlift.bytecode.Parameter; +import io.airlift.bytecode.ParameterizedType; +import io.airlift.bytecode.Scope; +import io.airlift.bytecode.Variable; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import static com.facebook.presto.operator.project.PageFieldsToInputParametersRewriter.rewritePageFieldsToInputParameters; +import static com.facebook.presto.spi.StandardErrorCode.COMPILER_ERROR; +import static com.facebook.presto.sql.gen.BytecodeUtils.invoke; +import static com.facebook.presto.sql.gen.LambdaExpressionExtractor.extractLambdaExpressions; +import static com.facebook.presto.util.CompilerUtils.defineClass; +import static com.facebook.presto.util.CompilerUtils.makeClassName; +import static io.airlift.bytecode.Access.FINAL; +import static io.airlift.bytecode.Access.PUBLIC; +import static io.airlift.bytecode.Access.a; +import static io.airlift.bytecode.BytecodeUtils.dumpBytecodeTree; +import static io.airlift.bytecode.Parameter.arg; +import static io.airlift.bytecode.ParameterizedType.type; +import static io.airlift.bytecode.expression.BytecodeExpressions.and; +import static io.airlift.bytecode.expression.BytecodeExpressions.constantBoolean; +import static io.airlift.bytecode.expression.BytecodeExpressions.constantFalse; +import static io.airlift.bytecode.expression.BytecodeExpressions.constantInt; +import static io.airlift.bytecode.expression.BytecodeExpressions.not; +import static java.util.Objects.requireNonNull; + +public class RowExpressionPredicateCompiler + implements PredicateCompiler +{ + private final Metadata metadata; + private final DeterminismEvaluator determinismEvaluator; + + private final LoadingCache> filterCache; + + @Inject + public RowExpressionPredicateCompiler(Metadata metadata) + { + this(metadata, 10_000); + } + + public RowExpressionPredicateCompiler(Metadata metadata, int filterCacheSize) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + this.determinismEvaluator = new DeterminismEvaluator(metadata.getFunctionManager()); + + if (filterCacheSize > 0) { + filterCache = CacheBuilder.newBuilder() + .recordStats() + .maximumSize(filterCacheSize) + .build(CacheLoader.from(filter -> compileFilterInternal(filter, Optional.empty()))); + } + else { + filterCache = null; + } + } + + @Override + public Supplier compilePredicate(RowExpression predicate, Optional classNameSuffix) + { + if (filterCache == null) { + return compileFilterInternal(predicate, classNameSuffix); + } + return filterCache.getUnchecked(predicate); + } + + private Supplier compileFilterInternal(RowExpression filter, Optional classNameSuffix) + { + requireNonNull(filter, "filter is null"); + + PageFieldsToInputParametersRewriter.Result result = rewritePageFieldsToInputParameters(filter); + int[] inputChannels = result.getInputChannels().getInputChannels().stream() + .mapToInt(Integer::intValue) + .toArray(); + + CallSiteBinder callSiteBinder = new CallSiteBinder(); + ClassDefinition classDefinition = defineFilterClass(result.getRewrittenExpression(), inputChannels, callSiteBinder, classNameSuffix); + + Class filterClass; + try { + filterClass = defineClass(classDefinition, Predicate.class, callSiteBinder.getBindings(), getClass().getClassLoader()); + } + catch (Exception e) { + throw new PrestoException(COMPILER_ERROR, filter.toString(), e.getCause()); + } + + System.out.println(dumpBytecodeTree(classDefinition)); + + return () -> { + try { + return filterClass.getConstructor().newInstance(); + } + catch (ReflectiveOperationException e) { + throw new PrestoException(COMPILER_ERROR, e); + } + }; + } + + private static ParameterizedType generateFilterClassName(Optional classNameSuffix) + { + return makeClassName(Predicate.class.getSimpleName(), classNameSuffix); + } + + private ClassDefinition defineFilterClass(RowExpression filter, int[] inputChannels, CallSiteBinder callSiteBinder, Optional classNameSuffix) + { + ClassDefinition classDefinition = new ClassDefinition( + a(PUBLIC, FINAL), + generateFilterClassName(classNameSuffix), + type(Object.class), + type(Predicate.class)); + + CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder); + + generateFilterMethod(classDefinition, callSiteBinder, cachedInstanceBinder, filter); + + // getInputChannels + classDefinition.declareMethod(a(PUBLIC), "getInputChannels", type(int[].class)) + .getBody() + .append(invoke(callSiteBinder.bind(inputChannels, int[].class), "getInputChannels")) + .retObject(); + + // isDeterministic + classDefinition.declareMethod(a(PUBLIC), "isDeterministic", type(boolean.class)) + .getBody() + .append(constantBoolean(determinismEvaluator.isDeterministic(filter))) + .retBoolean(); + + // constructor + generateConstructor(classDefinition, cachedInstanceBinder, method -> {}); + + return classDefinition; + } + + private MethodDefinition generateFilterMethod( + ClassDefinition classDefinition, + CallSiteBinder callSiteBinder, + CachedInstanceBinder cachedInstanceBinder, + RowExpression filter) + { + Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, filter); + + Parameter session = arg("session", ConnectorSession.class); + Parameter page = arg("page", Page.class); + Parameter position = arg("position", int.class); + + MethodDefinition method = classDefinition.declareMethod( + a(PUBLIC), + "test", + type(boolean.class), + ImmutableList.builder() + .add(session) + .add(page) + .add(position) + .build()); + + method.comment("Filter: %s", filter.toString()); + + Scope scope = method.getScope(); + BytecodeBlock body = method.getBody(); + + declareBlockVariables(filter, page, scope, body); + + Variable wasNullVariable = scope.declareVariable("wasNull", body, constantFalse()); + RowExpressionCompiler compiler = new RowExpressionCompiler( + callSiteBinder, + cachedInstanceBinder, + fieldReferenceCompiler(callSiteBinder), + metadata.getFunctionManager(), + compiledLambdaMap); + + Variable result = scope.declareVariable(boolean.class, "result"); + body.append(compiler.compile(filter, scope, Optional.empty())) + // store result so we can check for null + .putVariable(result) + .append(and(not(wasNullVariable), result).ret()); + return method; + } + + private Map generateMethodsForLambda( + ClassDefinition containerClassDefinition, + CallSiteBinder callSiteBinder, + CachedInstanceBinder cachedInstanceBinder, + RowExpression expression) + { + Set lambdaExpressions = ImmutableSet.copyOf(extractLambdaExpressions(expression)); + ImmutableMap.Builder compiledLambdaMap = ImmutableMap.builder(); + + int counter = 0; + for (LambdaDefinitionExpression lambdaExpression : lambdaExpressions) { + LambdaBytecodeGenerator.CompiledLambda compiledLambda = LambdaBytecodeGenerator.preGenerateLambdaExpression( + lambdaExpression, + "lambda_" + counter, + containerClassDefinition, + compiledLambdaMap.build(), + callSiteBinder, + cachedInstanceBinder, + metadata.getFunctionManager()); + compiledLambdaMap.put(lambdaExpression, compiledLambda); + counter++; + } + + return compiledLambdaMap.build(); + } + + private static void declareBlockVariables(RowExpression expression, Parameter page, Scope scope, BytecodeBlock body) + { + for (int channel : getInputChannels(expression)) { + scope.declareVariable("block_" + channel, body, page.invoke("getBlock", Block.class, constantInt(channel))); + } + } + + private static List getInputChannels(Iterable expressions) + { + TreeSet channels = new TreeSet<>(); + for (RowExpression expression : Expressions.subExpressions(expressions)) { + if (expression instanceof InputReferenceExpression) { + channels.add(((InputReferenceExpression) expression).getField()); + } + } + return ImmutableList.copyOf(channels); + } + + private static List getInputChannels(RowExpression expression) + { + return getInputChannels(ImmutableList.of(expression)); + } + + private static RowExpressionVisitor fieldReferenceCompiler(CallSiteBinder callSiteBinder) + { + return new InputReferenceCompiler( + (scope, field) -> scope.getVariable("block_" + field), + (scope, field) -> scope.getVariable("position"), + callSiteBinder); + } + + private static void generateConstructor( + ClassDefinition classDefinition, + CachedInstanceBinder cachedInstanceBinder, + Consumer additionalStatements) + { + MethodDefinition constructorDefinition = classDefinition.declareConstructor(a(PUBLIC)); + + BytecodeBlock body = constructorDefinition.getBody(); + Variable thisVariable = constructorDefinition.getThis(); + + body.comment("super();") + .append(thisVariable) + .invokeConstructor(Object.class); + + additionalStatements.accept(constructorDefinition); + + cachedInstanceBinder.generateInitializations(thisVariable, body); + body.ret(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java index e6ef335a0ea98..1a2db01207f96 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java @@ -85,6 +85,7 @@ import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.toList; +// No need to inherit from DomainTranslator; the class will be deprecated in the long term @Deprecated public final class ExpressionDomainTranslator { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionInterpreter.java index e066ae15dcdaa..c789c31ae4eeb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionInterpreter.java @@ -16,7 +16,6 @@ import com.facebook.presto.Session; import com.facebook.presto.client.FailureInfo; import com.facebook.presto.execution.warnings.WarningCollector; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.operator.scalar.ArraySubscriptOperator; import com.facebook.presto.spi.ConnectorSession; @@ -24,6 +23,7 @@ import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.RowBlockBuilder; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.FunctionType; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionNodeInliner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionNodeInliner.java index b4363972b5171..026e397728cba 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionNodeInliner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionNodeInliner.java @@ -19,6 +19,7 @@ import java.util.Map; +@Deprecated public class ExpressionNodeInliner extends ExpressionRewriter { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LazySplitSource.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LazySplitSource.java new file mode 100644 index 0000000000000..2b3fa2dbc68b7 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LazySplitSource.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.connector.ConnectorId; +import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.spi.connector.ConnectorPartitionHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.split.SplitSource; +import com.google.common.util.concurrent.ListenableFuture; + +import javax.annotation.concurrent.GuardedBy; + +import java.util.function.Supplier; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class LazySplitSource + implements SplitSource +{ + private final Supplier supplier; + + @GuardedBy("this") + private SplitSource delegate; + @GuardedBy("this") + private boolean closed; + + public LazySplitSource(Supplier supplier) + { + this.supplier = requireNonNull(supplier, "supplier is null"); + } + + @Override + public ConnectorId getConnectorId() + { + return getDelegate().getConnectorId(); + } + + @Override + public ConnectorTransactionHandle getTransactionHandle() + { + return getDelegate().getTransactionHandle(); + } + + @Override + public ListenableFuture getNextBatch(ConnectorPartitionHandle partitionHandle, Lifespan lifespan, int maxSize) + { + return getDelegate().getNextBatch(partitionHandle, lifespan, maxSize); + } + + @Override + public boolean isFinished() + { + return getDelegate().isFinished(); + } + + @Override + public synchronized void close() + { + // already closed + if (closed) { + return; + } + closed = true; + + // not yet initialized + if (delegate == null) { + return; + } + + delegate.close(); + } + + private synchronized SplitSource getDelegate() + { + checkState(!closed, "split source is closed"); + if (delegate == null) { + delegate = supplier.get(); + } + return delegate; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java index 2a5054164fd76..ee935a5494da6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java @@ -52,7 +52,6 @@ import com.facebook.presto.sql.tree.Literal; import com.facebook.presto.sql.tree.LongLiteral; import com.facebook.presto.sql.tree.NullLiteral; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.StringLiteral; import com.facebook.presto.sql.tree.TimeLiteral; import com.facebook.presto.sql.tree.TimestampLiteral; @@ -238,7 +237,7 @@ protected Object visitGenericLiteral(GenericLiteral node, ConnectorSession sessi } if (JSON.equals(type)) { - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(QualifiedName.of("json_parse"), fromTypes(VARCHAR)); + FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction("json_parse", fromTypes(VARCHAR)); return functionInvoker.invoke(functionHandle, session, ImmutableList.of(utf8Slice(node.getValue()))); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index a48143df2a70c..b1c5f4dd93f6a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -1205,18 +1205,8 @@ private PhysicalOperation visitScanFilterAndProject( // filterExpression may contain large function calls; evaluate them before compiling. if (filterExpression.isPresent()) { - Type type = filterExpression.get().getType(); - Object value = new RowExpressionInterpreter(filterExpression.get(), metadata, session, true).optimize(); - if (value instanceof RowExpression) { - RowExpression optimizedFilter = (RowExpression) value; - // building channel info - optimizedFilter = SymbolToChannelTranslator.translate(optimizedFilter, sourceLayout); - filterExpression = Optional.of(optimizedFilter); - } - else { - // TODO: theoretically value should either be true or false; we could possibly optimized the execution based on this - filterExpression = Optional.of(constant(value, type)); - } + // TODO: theoretically, filterExpression could be a constant value (true or false) after optimization; we could possibly optimize the execution. + filterExpression = Optional.of(bindChannels(filterExpression.get(), sourceLayout)); } // build output mapping @@ -1286,11 +1276,27 @@ private PhysicalOperation visitScanFilterAndProject( } } + // TODO: migrate `toRowExpression` to `bindChannels` private RowExpression toRowExpression(Expression expression, Map, Type> types, Map sourceLayout) { return SqlToRowExpressionTranslator.translate(expression, types, sourceLayout, metadata.getFunctionManager(), metadata.getTypeManager(), session, true); } + private RowExpression bindChannels(RowExpression expression, Map sourceLayout) + { + Type type = expression.getType(); + Object value = new RowExpressionInterpreter(expression, metadata, session.toConnectorSession(), true).optimize(); + if (value instanceof RowExpression) { + RowExpression optimized = (RowExpression) value; + // building channel info + expression = SymbolToChannelTranslator.translate(optimized, sourceLayout); + } + else { + expression = constant(value, type); + } + return expression; + } + @Override public PhysicalOperation visitTableScan(TableScanNode node, LocalExecutionPlanContext context) { @@ -1320,7 +1326,7 @@ public PhysicalOperation visitValues(ValuesNode node, LocalExecutionPlanContext pageBuilder.declarePosition(); for (int i = 0; i < row.size(); i++) { // evaluate the literal value - Object result = rowExpressionInterpreter(row.get(i), metadata, context.getSession()).evaluate(); + Object result = rowExpressionInterpreter(row.get(i), metadata, context.getSession().toConnectorSession()).evaluate(); writeNativeValue(outputTypes.get(i), pageBuilder.getBlockBuilder(i), result); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitionMap.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitionMap.java index b2f59e3c70b79..6e3f05eb4e002 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitionMap.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitionMap.java @@ -15,8 +15,8 @@ import com.facebook.presto.execution.scheduler.BucketNodeMap; import com.facebook.presto.execution.scheduler.FixedBucketNodeMap; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableList; import java.util.List; @@ -37,25 +37,25 @@ // in the above case, as the co-existence of BucketNodeMap and NodePartitionMap is confusing. public class NodePartitionMap { - private final List partitionToNode; + private final List partitionToNode; private final int[] bucketToPartition; private final ToIntFunction splitToBucket; - public NodePartitionMap(List partitionToNode, ToIntFunction splitToBucket) + public NodePartitionMap(List partitionToNode, ToIntFunction splitToBucket) { this.partitionToNode = ImmutableList.copyOf(requireNonNull(partitionToNode, "partitionToNode is null")); this.bucketToPartition = IntStream.range(0, partitionToNode.size()).toArray(); this.splitToBucket = requireNonNull(splitToBucket, "splitToBucket is null"); } - public NodePartitionMap(List partitionToNode, int[] bucketToPartition, ToIntFunction splitToBucket) + public NodePartitionMap(List partitionToNode, int[] bucketToPartition, ToIntFunction splitToBucket) { this.bucketToPartition = requireNonNull(bucketToPartition, "bucketToPartition is null"); this.partitionToNode = ImmutableList.copyOf(requireNonNull(partitionToNode, "partitionToNode is null")); this.splitToBucket = requireNonNull(splitToBucket, "splitToBucket is null"); } - public List getPartitionToNode() + public List getPartitionToNode() { return partitionToNode; } @@ -65,7 +65,7 @@ public int[] getBucketToPartition() return bucketToPartition; } - public Node getNode(Split split) + public InternalNode getNode(Split split) { int bucket = splitToBucket.applyAsInt(split); int partition = bucketToPartition[bucket]; @@ -74,7 +74,7 @@ public Node getNode(Split split) public BucketNodeMap asBucketNodeMap() { - ImmutableList.Builder bucketToNode = ImmutableList.builder(); + ImmutableList.Builder bucketToNode = ImmutableList.builder(); for (int partition : bucketToPartition) { bucketToNode.add(partitionToNode.get(partition)); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java index 888b1856f5bfc..fee4bced50184 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java @@ -19,12 +19,12 @@ import com.facebook.presto.execution.scheduler.FixedBucketNodeMap; import com.facebook.presto.execution.scheduler.NodeScheduler; import com.facebook.presto.execution.scheduler.group.DynamicBucketNodeMap; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.BucketPartitionFunction; import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.spi.BucketFunction; import com.facebook.presto.spi.ConnectorSplit; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.connector.ConnectorBucketNodeMap; import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; @@ -138,9 +138,9 @@ public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHand // safety check for crazy partitioning checkArgument(connectorBucketNodeMap.getBucketCount() < 1_000_000, "Too many buckets in partitioning: %s", connectorBucketNodeMap.getBucketCount()); - List bucketToNode; + List bucketToNode; if (connectorBucketNodeMap.hasFixedMapping()) { - bucketToNode = connectorBucketNodeMap.getFixedMapping(); + bucketToNode = getFixedMapping(connectorBucketNodeMap); } else { bucketToNode = createArbitraryBucketToNode( @@ -149,10 +149,10 @@ public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHand } int[] bucketToPartition = new int[connectorBucketNodeMap.getBucketCount()]; - BiMap nodeToPartition = HashBiMap.create(); + BiMap nodeToPartition = HashBiMap.create(); int nextPartitionId = 0; for (int bucket = 0; bucket < bucketToNode.size(); bucket++) { - Node node = bucketToNode.get(bucket); + InternalNode node = bucketToNode.get(bucket); Integer partitionId = nodeToPartition.get(node); if (partitionId == null) { partitionId = nextPartitionId++; @@ -161,7 +161,7 @@ public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHand bucketToPartition[bucket] = partitionId; } - List partitionToNode = IntStream.range(0, nodeToPartition.size()) + List partitionToNode = IntStream.range(0, nodeToPartition.size()) .mapToObj(partitionId -> nodeToPartition.inverse().get(partitionId)) .collect(toImmutableList()); @@ -173,7 +173,7 @@ public BucketNodeMap getBucketNodeMap(Session session, PartitioningHandle partit ConnectorBucketNodeMap connectorBucketNodeMap = getConnectorBucketNodeMap(session, partitioningHandle); if (connectorBucketNodeMap.hasFixedMapping()) { - return new FixedBucketNodeMap(getSplitToBucket(session, partitioningHandle), connectorBucketNodeMap.getFixedMapping()); + return new FixedBucketNodeMap(getSplitToBucket(session, partitioningHandle), getFixedMapping(connectorBucketNodeMap)); } if (preferDynamic) { @@ -187,6 +187,13 @@ public BucketNodeMap getBucketNodeMap(Session session, PartitioningHandle partit connectorBucketNodeMap.getBucketCount())); } + private static List getFixedMapping(ConnectorBucketNodeMap connectorBucketNodeMap) + { + return connectorBucketNodeMap.getFixedMapping().stream() + .map(InternalNode.class::cast) + .collect(toImmutableList()); + } + private ConnectorBucketNodeMap getConnectorBucketNodeMap(Session session, PartitioningHandle partitioningHandle) { checkArgument(!(partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle)); @@ -229,7 +236,7 @@ private ToIntFunction getSplitToBucket(Session session, PartitioningHandl }; } - private static List createArbitraryBucketToNode(List nodes, int bucketCount) + private static List createArbitraryBucketToNode(List nodes, int bucketCount) { return cyclingShuffledStream(nodes) .limit(bucketCount) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java index 9fde9e440083f..3194f1ab0ad35 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java @@ -560,7 +560,8 @@ private TableScanNode createTemporaryTableScan( assignments, Optional.of(layoutHandle), TupleDomain.all(), - TupleDomain.all()); + TupleDomain.all(), + true); } private TableFinishNode createTemporaryTableWrite( @@ -1074,7 +1075,8 @@ public PlanNode visitTableScan(TableScanNode node, RewriteContext context) node.getAssignments(), Optional.of(newTableLayoutHandle), node.getCurrentConstraint(), - node.getEnforcedConstraint()); + node.getEnforcedConstraint(), + node.isTemporaryTable()); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index 72984ea1c89fb..098a9cf0979b0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.sql.planner; -import com.facebook.presto.Session; import com.facebook.presto.client.FailureInfo; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.RowBlockBuilder; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; @@ -36,8 +36,8 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.InterpretedFunctionInvoker; import com.facebook.presto.sql.planner.Interpreters.LambdaSymbolResolver; -import com.facebook.presto.sql.relational.StandardFunctionResolution; -import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.optimizer.ExpressionOptimizer; import com.facebook.presto.util.Failures; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -103,15 +103,15 @@ public class RowExpressionInterpreter { private final RowExpression expression; private final Metadata metadata; - private final Session session; + private final ConnectorSession session; private final boolean optimize; private final InterpretedFunctionInvoker functionInvoker; private final com.facebook.presto.sql.relational.DeterminismEvaluator determinismEvaluator; - private final StandardFunctionResolution resolution; + private final FunctionResolution resolution; private final Visitor visitor; - public static Object evaluateConstantRowExpression(RowExpression expression, Metadata metadata, Session session) + public static Object evaluateConstantRowExpression(RowExpression expression, Metadata metadata, ConnectorSession session) { // evaluate the expression Object result = new RowExpressionInterpreter(expression, metadata, session, false).evaluate(); @@ -119,12 +119,12 @@ public static Object evaluateConstantRowExpression(RowExpression expression, Met return result; } - public static RowExpressionInterpreter rowExpressionInterpreter(RowExpression expression, Metadata metadata, Session session) + public static RowExpressionInterpreter rowExpressionInterpreter(RowExpression expression, Metadata metadata, ConnectorSession session) { return new RowExpressionInterpreter(expression, metadata, session, false); } - public RowExpressionInterpreter(RowExpression expression, Metadata metadata, Session session, boolean optimize) + public RowExpressionInterpreter(RowExpression expression, Metadata metadata, ConnectorSession session, boolean optimize) { this.expression = requireNonNull(expression, "expression is null"); this.metadata = requireNonNull(metadata, "metadata is null"); @@ -132,7 +132,7 @@ public RowExpressionInterpreter(RowExpression expression, Metadata metadata, Ses this.optimize = optimize; this.functionInvoker = new InterpretedFunctionInvoker(metadata.getFunctionManager()); this.determinismEvaluator = new com.facebook.presto.sql.relational.DeterminismEvaluator(metadata.getFunctionManager()); - this.resolution = new StandardFunctionResolution(metadata.getFunctionManager()); + this.resolution = new FunctionResolution(metadata.getFunctionManager()); this.visitor = new Visitor(); } @@ -151,7 +151,7 @@ public Object evaluate() public Object optimize() { checkState(optimize, "optimize() not allowed for interpreter"); - return expression.accept(visitor, null); + return optimize(null); } /** @@ -161,7 +161,13 @@ public Object optimize() public Object optimize(SymbolResolver inputs) { checkState(optimize, "optimize(SymbolResolver) not allowed for interpreter"); - return expression.accept(visitor, inputs); + Object result = expression.accept(visitor, inputs); + + if (!(result instanceof RowExpression)) { + // constant folding + return result; + } + return new ExpressionOptimizer(metadata.getFunctionManager(), session).optimize((RowExpression) result); } private class Visitor @@ -237,7 +243,7 @@ public Object visitCall(CallExpression node, Object context) if (optimize && (!functionMetadata.isDeterministic() || hasUnresolvedValue(argumentValues) || functionMetadata.getName().equals("fail"))) { return call(node.getDisplayName(), functionHandle, node.getType(), toRowExpressions(argumentValues, argumentTypes)); } - return functionInvoker.invoke(functionHandle, session.toConnectorSession(), argumentValues); + return functionInvoker.invoke(functionHandle, session, argumentValues); } @Override @@ -317,8 +323,8 @@ else if (Boolean.TRUE.equals(condition)) { EQUAL, ImmutableList.of(commonType, commonType), ImmutableList.of( - functionInvoker.invoke(firstCast, session.toConnectorSession(), left), - functionInvoker.invoke(secondCast, session.toConnectorSession(), right)))); + functionInvoker.invoke(firstCast, session, left), + functionInvoker.invoke(secondCast, session, right)))); if (equal) { return null; @@ -633,10 +639,10 @@ private RowExpression createFailureFunction(RuntimeException exception, Type typ requireNonNull(exception, "Exception is null"); String failureInfo = JsonCodec.jsonCodec(FailureInfo.class).toJson(Failures.toFailure(exception).toFailureInfo()); - FunctionHandle jsonParse = metadata.getFunctionManager().resolveFunction(session, QualifiedName.of("json_parse"), fromTypes(VARCHAR)); - Object json = functionInvoker.invoke(jsonParse, session.toConnectorSession(), utf8Slice(failureInfo)); + FunctionHandle jsonParse = metadata.getFunctionManager().lookupFunction("json_parse", fromTypes(VARCHAR)); + Object json = functionInvoker.invoke(jsonParse, session, utf8Slice(failureInfo)); - FunctionHandle failureFunction = metadata.getFunctionManager().resolveFunction(session, QualifiedName.of("fail"), fromTypes(JSON)); + FunctionHandle failureFunction = metadata.getFunctionManager().lookupFunction("fail", fromTypes(JSON)); FunctionHandle cast = metadata.getFunctionManager().lookupCast(CAST, UNKNOWN.getTypeSignature(), type.getTypeSignature()); return call(CAST.name(), cast, type, call("fail", failureFunction, UNKNOWN, toRowExpression(json, JSON))); @@ -655,7 +661,7 @@ private boolean hasUnresolvedValue(List values) private Object invokeOperator(OperatorType operatorType, List argumentTypes, List argumentValues) { FunctionHandle operatorHandle = metadata.getFunctionManager().resolveOperator(operatorType, fromTypes(argumentTypes)); - return functionInvoker.invoke(operatorHandle, session.toConnectorSession(), argumentValues); + return functionInvoker.invoke(operatorHandle, session, argumentValues); } private List toRowExpressions(List values, List types) @@ -720,7 +726,7 @@ private SpecialCallResult tryHandleCast(CallExpression callExpression, List argumentValues, List argumentTypes, Object context) { - StandardFunctionResolution resolution = new StandardFunctionResolution(metadata.getFunctionManager()); + FunctionResolution resolution = new FunctionResolution(metadata.getFunctionManager()); checkArgument(resolution.isLikeFunction(callExpression.getFunctionHandle())); checkArgument(callExpression.getArguments().size() == 2); RowExpression likePatternExpression = callExpression.getArguments().get(1); @@ -763,7 +769,7 @@ private SpecialCallResult tryHandleLike(CallExpression callExpression, List visitExplainAnalyze(ExplainAnalyzeNode node, public Map visitTableScan(TableScanNode node, Void context) { // get dataSource for table - SplitSource splitSource = splitSourceProvider.getSplits( + Supplier splitSourceSupplier = () -> splitSourceProvider.getSplits( session, node.getLayout().get(), stageExecutionDescriptor.isScanGroupedExecution(node.getId()) ? GROUPED_SCHEDULING : UNGROUPED_SCHEDULING); + SplitSource splitSource = node.isTemporaryTable() ? new LazySplitSource(splitSourceSupplier) : splitSourceSupplier.get(); + splitSources.add(splitSource); return ImmutableMap.of(node.getId(), splitSource); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java index dc78bca690d10..cb28d03317ff7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java @@ -80,9 +80,9 @@ public TableStatisticAggregation createStatisticsAggregation(TableStatisticsMeta if (type != ROW_COUNT) { throw new PrestoException(NOT_SUPPORTED, "Table-wide statistic type not supported: " + type); } - QualifiedName count = QualifiedName.of("count"); + String count = "count"; AggregationNode.Aggregation aggregation = new AggregationNode.Aggregation( - new FunctionCall(count, ImmutableList.of()), + new FunctionCall(QualifiedName.of(count), ImmutableList.of()), functionManager.lookupFunction(count, ImmutableList.of()), Optional.empty()); Symbol symbol = symbolAllocator.newSymbol("rowCount", BIGINT); @@ -111,25 +111,25 @@ private ColumnStatisticsAggregation createColumnAggregation(ColumnStatisticType { switch (statisticType) { case MIN_VALUE: - return createAggregation(QualifiedName.of("min"), input.toSymbolReference(), inputType, inputType); + return createAggregation("min", input.toSymbolReference(), inputType, inputType); case MAX_VALUE: - return createAggregation(QualifiedName.of("max"), input.toSymbolReference(), inputType, inputType); + return createAggregation("max", input.toSymbolReference(), inputType, inputType); case NUMBER_OF_DISTINCT_VALUES: - return createAggregation(QualifiedName.of("approx_distinct"), input.toSymbolReference(), inputType, BIGINT); + return createAggregation("approx_distinct", input.toSymbolReference(), inputType, BIGINT); case NUMBER_OF_NON_NULL_VALUES: - return createAggregation(QualifiedName.of("count"), input.toSymbolReference(), inputType, BIGINT); + return createAggregation("count", input.toSymbolReference(), inputType, BIGINT); case NUMBER_OF_TRUE_VALUES: - return createAggregation(QualifiedName.of("count_if"), input.toSymbolReference(), BOOLEAN, BIGINT); + return createAggregation("count_if", input.toSymbolReference(), BOOLEAN, BIGINT); case TOTAL_SIZE_IN_BYTES: - return createAggregation(QualifiedName.of(SumDataSizeForStats.NAME), input.toSymbolReference(), inputType, BIGINT); + return createAggregation(SumDataSizeForStats.NAME, input.toSymbolReference(), inputType, BIGINT); case MAX_VALUE_SIZE_IN_BYTES: - return createAggregation(QualifiedName.of(MaxDataSizeForStats.NAME), input.toSymbolReference(), inputType, BIGINT); + return createAggregation(MaxDataSizeForStats.NAME, input.toSymbolReference(), inputType, BIGINT); default: throw new IllegalArgumentException("Unsupported statistic type: " + statisticType); } } - private ColumnStatisticsAggregation createAggregation(QualifiedName functionName, SymbolReference input, Type inputType, Type outputType) + private ColumnStatisticsAggregation createAggregation(String functionName, SymbolReference input, Type inputType, Type outputType) { FunctionManager functionManager = metadata.getFunctionManager(); FunctionHandle functionHandle = functionManager.lookupFunction(functionName, TypeSignatureProvider.fromTypes(ImmutableList.of(inputType))); @@ -137,7 +137,7 @@ private ColumnStatisticsAggregation createAggregation(QualifiedName functionName verify(resolvedType.equals(inputType), "resolved function input type does not match the input type: %s != %s", resolvedType, inputType); return new ColumnStatisticsAggregation( new AggregationNode.Aggregation( - new FunctionCall(functionName, ImmutableList.of(input)), + new FunctionCall(QualifiedName.of(functionName), ImmutableList.of(input)), functionHandle, Optional.empty()), outputType); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubPlan.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubPlan.java index 30e9e45cc33fa..8929d0096e9ee 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubPlan.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubPlan.java @@ -21,7 +21,6 @@ import javax.annotation.concurrent.Immutable; import java.util.List; -import java.util.Optional; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableMultiset.toImmutableMultiset; @@ -67,11 +66,6 @@ public List getAllFragments() return fragments.build(); } - public SubPlan withBucketToPartition(Optional bucketToPartition) - { - return new SubPlan(fragment.withBucketToPartition(bucketToPartition), children); - } - public void sanityCheck() { Multiset remoteSourceIds = fragment.getRemoteSourceNodes().stream() diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java index f3506252b1819..0b2bf4fd26f15 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java @@ -57,6 +57,7 @@ import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.sql.analyzer.SemanticExceptions.notSupportedException; +import static com.facebook.presto.sql.analyzer.SemanticExceptions.subQueryNotSupportedError; import static com.facebook.presto.sql.planner.ExpressionNodeInliner.replaceExpression; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; @@ -250,7 +251,7 @@ public PlanBuilder appendLateralJoin(PlanBuilder subPlan, PlanBuilder subqueryPl subqueryNode, ImmutableList.copyOf(SymbolsExtractor.extractUnique(correlation.values())), type, - query), + subQueryNotSupportedError(query, "Given correlated subquery")), analysis.getParameters()); } @@ -427,12 +428,7 @@ private List coercionsFor(Expression expression) .collect(toImmutableList()); } - private PlanBuilder appendApplyNode( - PlanBuilder subPlan, - Node subquery, - PlanNode subqueryNode, - Assignments subqueryAssignments, - boolean correlationAllowed) + private PlanBuilder appendApplyNode(PlanBuilder subPlan, Node subquery, PlanNode subqueryNode, Assignments subqueryAssignments, boolean correlationAllowed) { Map correlation = extractCorrelation(subPlan, subqueryNode); if (!correlationAllowed && !correlation.isEmpty()) { @@ -449,7 +445,7 @@ private PlanBuilder appendApplyNode( subqueryNode, subqueryAssignments, ImmutableList.copyOf(SymbolsExtractor.extractUnique(correlation.values())), - subquery), + subQueryNotSupportedError(subquery, "Given correlated subquery")), analysis.getParameters()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java index 561c183e5d9c7..91e531b69bb64 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.spi.relation.DefaultRowExpressionTraversalVisitor; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.plan.PlanNode; -import com.facebook.presto.sql.relational.DefaultRowExpressionTraversalVisitor; import com.facebook.presto.sql.tree.DefaultExpressionTraversalVisitor; import com.facebook.presto.sql.tree.DefaultTraversalVisitor; import com.facebook.presto.sql.tree.DereferenceExpression; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java index 0f468b755131a..fe273ba2e729c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java @@ -16,13 +16,13 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.scheduler.NodeScheduler; import com.facebook.presto.execution.scheduler.NodeSelector; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.operator.BucketPartitionFunction; import com.facebook.presto.operator.HashGenerator; import com.facebook.presto.operator.InterpretedHashGenerator; import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.operator.PrecomputedHashGenerator; import com.facebook.presto.spi.BucketFunction; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; import com.facebook.presto.spi.type.Type; @@ -139,7 +139,7 @@ public String toString() public NodePartitionMap getNodePartitionMap(Session session, NodeScheduler nodeScheduler) { NodeSelector nodeSelector = nodeScheduler.createNodeSelector(null); - List nodes; + List nodes; if (partitioning == SystemPartitioning.COORDINATOR_ONLY) { nodes = ImmutableList.of(nodeSelector.selectCurrentNode()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java index c5216f5ac3bad..c8b0335705608 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java @@ -321,7 +321,7 @@ public Result apply(ApplyNode applyNode, Captures captures, Context context) applyNode.getSubquery(), subqueryAssignments, applyNode.getCorrelation(), - applyNode.getOriginSubquery())); + applyNode.getOriginSubqueryError())); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java index 10e49c3d478c6..5b983842dac78 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java @@ -107,7 +107,7 @@ public Result apply(AggregationNode node, Captures captures, Context context) aggregations.put(entry.getKey(), new Aggregation( new FunctionCall(name, ImmutableList.of(envelopeSymbol.toSymbolReference(), partitionCountSymbol.toSymbolReference())), - metadata.getFunctionManager().lookupFunction(QualifiedName.of(NAME), fromTypes(geometryType, INTEGER)), + metadata.getFunctionManager().lookupFunction(NAME, fromTypes(geometryType, INTEGER)), aggregation.getMask())); } else { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java index a83f3cb23b674..8564296c4c372 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java @@ -77,7 +77,7 @@ public Result apply(AggregationNode parent, Captures captures, Context context) changed = true; aggregations.put(symbol, new AggregationNode.Aggregation( new FunctionCall(QualifiedName.of("count"), ImmutableList.of()), - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of()), + functionManager.lookupFunction("count", ImmutableList.of()), aggregation.getMask())); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java index 26b015871d684..775e2a8db122d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java @@ -260,7 +260,7 @@ private AggregationNode.Aggregation countWithFilter(Expression condition) return new AggregationNode.Aggregation( countCall, - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of()), + functionManager.lookupFunction("count", ImmutableList.of()), Optional.empty()); /* mask */ } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarSubquery.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarSubquery.java index 0ae84f4ea7179..f25319afec143 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarSubquery.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarSubquery.java @@ -112,7 +112,7 @@ public Result apply(LateralJoinNode lateralJoinNode, Captures captures, Context rewrittenSubquery, lateralJoinNode.getCorrelation(), lateralJoinNode.getType(), - lateralJoinNode.getOriginSubquery())); + lateralJoinNode.getOriginSubqueryError())); } Symbol unique = context.getSymbolAllocator().newSymbol("unique", BigintType.BIGINT); @@ -126,7 +126,7 @@ public Result apply(LateralJoinNode lateralJoinNode, Captures captures, Context rewrittenSubquery, lateralJoinNode.getCorrelation(), lateralJoinNode.getType(), - lateralJoinNode.getOriginSubquery()); + lateralJoinNode.getOriginSubqueryError()); Symbol isDistinct = context.getSymbolAllocator().newSymbol("is_distinct", BooleanType.BOOLEAN); MarkDistinctNode markDistinctNode = new MarkDistinctNode( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java index 3257b2c2f807c..1ee7f1d3e46bc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java @@ -78,8 +78,8 @@ public class TransformExistsApplyToLateralNode { private static final Pattern PATTERN = applyNode(); - private static final QualifiedName COUNT = QualifiedName.of("count"); - private static final FunctionCall COUNT_CALL = new FunctionCall(COUNT, ImmutableList.of()); + private static final String COUNT = "count"; + private static final FunctionCall COUNT_CALL = new FunctionCall(QualifiedName.of(COUNT), ImmutableList.of()); private final FunctionManager functionManager; @@ -144,13 +144,13 @@ private Optional rewriteToNonDefaultAggregation(ApplyNode applyNode, C subquery, applyNode.getCorrelation(), LEFT, - applyNode.getOriginSubquery()), + applyNode.getOriginSubqueryError()), assignments.build())); } private PlanNode rewriteToDefaultAggregation(ApplyNode parent, Context context) { - Symbol count = context.getSymbolAllocator().newSymbol(COUNT.toString(), BIGINT); + Symbol count = context.getSymbolAllocator().newSymbol(COUNT, BIGINT); Symbol exists = getOnlyElement(parent.getSubqueryAssignments().getSymbols()); return new LateralJoinNode( @@ -173,6 +173,6 @@ private PlanNode rewriteToDefaultAggregation(ApplyNode parent, Context context) Assignments.of(exists, new ComparisonExpression(GREATER_THAN, count.toSymbolReference(), new Cast(new LongLiteral("0"), BIGINT.toString())))), parent.getCorrelation(), INNER, - parent.getOriginSubquery()); + parent.getOriginSubqueryError()); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java index 1d722155f49c8..44480f48f3e5d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java @@ -717,9 +717,12 @@ private PlanWithProperties planPartitionedJoin(JoinNode node, List leftS Partitioning rightPartitioning = left.getProperties().translate(createTranslator(leftToRight)).getNodePartitioning().get(); right = node.getRight().accept(this, PreferredProperties.partitioned(rightPartitioning)); if (!right.getProperties().isCompatibleTablePartitioningWith(left.getProperties(), rightToLeft::get, metadata, session)) { - // TODO: support materialization when one of the tables is pre-bucketed right = withDerivedProperties( - partitionedExchange(idAllocator.getNextId(), REMOTE_STREAMING, right.getNode(), new PartitioningScheme(rightPartitioning, right.getNode().getOutputSymbols())), + partitionedExchange( + idAllocator.getNextId(), + selectExchangeScopeForPartitionedRemoteExchange(right.getNode()), + right.getNode(), + new PartitioningScheme(rightPartitioning, right.getNode().getOutputSymbols())), right.getProperties()); } } @@ -728,9 +731,12 @@ private PlanWithProperties planPartitionedJoin(JoinNode node, List leftS if (right.getProperties().isNodePartitionedOn(rightSymbols) && !right.getProperties().isSingleNode()) { Partitioning leftPartitioning = right.getProperties().translate(createTranslator(rightToLeft)).getNodePartitioning().get(); - // TODO: support materialization when one of the tables is pre-bucketed left = withDerivedProperties( - partitionedExchange(idAllocator.getNextId(), REMOTE_STREAMING, left.getNode(), new PartitioningScheme(leftPartitioning, left.getNode().getOutputSymbols())), + partitionedExchange( + idAllocator.getNextId(), + selectExchangeScopeForPartitionedRemoteExchange(left.getNode()), + left.getNode(), + new PartitioningScheme(leftPartitioning, left.getNode().getOutputSymbols())), left.getProperties()); } else { @@ -758,9 +764,11 @@ private PlanWithProperties planPartitionedJoin(JoinNode node, List leftS // if colocated joins are disabled, force redistribute when using a custom partitioning if (!isColocatedJoinEnabled(session) && hasMultipleTableScans(left.getNode(), right.getNode())) { Partitioning rightPartitioning = left.getProperties().translate(createTranslator(leftToRight)).getNodePartitioning().get(); - // TODO: support materialization when one of the tables is pre-bucketed right = withDerivedProperties( - partitionedExchange(idAllocator.getNextId(), REMOTE_STREAMING, right.getNode(), new PartitioningScheme(rightPartitioning, right.getNode().getOutputSymbols())), + partitionedExchange( + idAllocator.getNextId(), + selectExchangeScopeForPartitionedRemoteExchange(right.getNode()), right.getNode(), + new PartitioningScheme(rightPartitioning, right.getNode().getOutputSymbols())), right.getProperties()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CheckSubqueryNodesAreRewritten.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CheckSubqueryNodesAreRewritten.java index 4b1263a737faf..7cb143f7ce203 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CheckSubqueryNodesAreRewritten.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CheckSubqueryNodesAreRewritten.java @@ -24,13 +24,12 @@ import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; import com.facebook.presto.sql.planner.plan.PlanNode; -import com.facebook.presto.sql.tree.Node; import java.util.List; -import static com.facebook.presto.sql.analyzer.SemanticExceptions.notSupportedException; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; public class CheckSubqueryNodesAreRewritten implements PlanOptimizer @@ -42,22 +41,22 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Sym .findFirst() .ifPresent(node -> { ApplyNode applyNode = (ApplyNode) node; - throw error(applyNode.getCorrelation(), applyNode.getOriginSubquery()); + throw error(applyNode.getCorrelation(), applyNode.getOriginSubqueryError()); }); searchFrom(plan).where(LateralJoinNode.class::isInstance) .findFirst() .ifPresent(node -> { LateralJoinNode lateralJoinNode = (LateralJoinNode) node; - throw error(lateralJoinNode.getCorrelation(), lateralJoinNode.getOriginSubquery()); + throw error(lateralJoinNode.getCorrelation(), lateralJoinNode.getOriginSubqueryError()); }); return plan; } - private SemanticException error(List correlation, Node originSubquery) + private SemanticException error(List correlation, String originSubqueryError) { checkState(!correlation.isEmpty(), "All the non correlated subqueries should be rewritten at this point"); - throw notSupportedException(originSubquery, "Given correlated subquery"); + throw new RuntimeException(format(originSubqueryError, "Given correlated subquery is not supported")); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java index cf32724b52ec6..7981203360f9d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java @@ -250,7 +250,7 @@ private AggregationNode computeCounts(UnionNode sourceNode, List origina QualifiedName name = QualifiedName.of("count"); aggregations.put(output, new Aggregation( new FunctionCall(name, ImmutableList.of(markers.get(i).toSymbolReference())), - functionManager.lookupFunction(name, fromTypes(BIGINT)), + functionManager.lookupFunction(name.getSuffix(), fromTypes(BIGINT)), Optional.empty())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java index 4cebb5f851d35..f97ce74878088 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java @@ -169,10 +169,9 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext inpu tupleDomain = ExpressionDomainTranslator.fromPredicate(metadata, session, castToExpression(node.getPredicate()), types).getTupleDomain(); } else { - tupleDomain = RowExpressionDomainTranslator.fromPredicate(metadata, session, node.getPredicate()) + tupleDomain = new RowExpressionDomainTranslator(metadata).fromPredicate(session.toConnectorSession(), node.getPredicate(), BASIC_COLUMN_EXTRACTOR) .getTupleDomain() .transform(column -> new Symbol(column.getName())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java index 03545578d69bd..71d172d057eb6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java @@ -801,7 +801,7 @@ public PlanNode visitApply(ApplyNode node, RewriteContext> context) .addAll(subqueryAssignmentsSymbols) // need to include those: e.g: "expr" from "expr IN (SELECT 1)" .build(); PlanNode input = context.rewrite(node.getInput(), inputContext); - return new ApplyNode(node.getId(), input, subquery, subqueryAssignments.build(), newCorrelation, node.getOriginSubquery()); + return new ApplyNode(node.getId(), input, subquery, subqueryAssignments.build(), newCorrelation, node.getOriginSubqueryError()); } @Override @@ -840,7 +840,7 @@ public PlanNode visitLateralJoin(LateralJoinNode node, RewriteContext createAggregationNode( COUNT, ImmutableList.of(nonNullableAggregationSourceSymbol.toSymbolReference())), functionManager.lookupFunction( - COUNT, + COUNT.getSuffix(), fromTypeSignatures(scalarAggregationSourceTypeSignatures)), entry.getValue().getMask())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java index 3639078862159..9d0720764ed42 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java @@ -148,19 +148,19 @@ private PlanNode rewriteQuantifiedApplyNode(ApplyNode node, QuantifiedComparison ImmutableMap.of( minValue, new Aggregation( new FunctionCall(MIN, outputColumnReferences), - functionManager.lookupFunction(MIN, outputColumnTypeSignatures), + functionManager.lookupFunction(MIN.getSuffix(), outputColumnTypeSignatures), Optional.empty()), maxValue, new Aggregation( new FunctionCall(MAX, outputColumnReferences), - functionManager.lookupFunction(MAX, outputColumnTypeSignatures), + functionManager.lookupFunction(MAX.getSuffix(), outputColumnTypeSignatures), Optional.empty()), countAllValue, new Aggregation( new FunctionCall(COUNT, emptyList()), - functionManager.lookupFunction(COUNT, emptyList()), + functionManager.lookupFunction(COUNT.getSuffix(), emptyList()), Optional.empty()), countNonNullValue, new Aggregation( new FunctionCall(COUNT, outputColumnReferences), - functionManager.lookupFunction(COUNT, outputColumnTypeSignatures), + functionManager.lookupFunction(COUNT.getSuffix(), outputColumnTypeSignatures), Optional.empty())), globalAggregation(), ImmutableList.of(), @@ -174,7 +174,7 @@ countNonNullValue, new Aggregation( subqueryPlan, node.getCorrelation(), LateralJoinNode.Type.INNER, - node.getOriginSubquery()); + node.getOriginSubqueryError()); Expression valueComparedToSubquery = rewriteUsingBounds(quantifiedComparison, minValue, maxValue, countAllValue, countNonNullValue); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java index 1f4db418d4950..45ead3d8f454f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java @@ -455,7 +455,7 @@ public PlanNode visitApply(ApplyNode node, RewriteContext context) PlanNode subquery = context.rewrite(node.getSubquery()); List canonicalCorrelation = Lists.transform(node.getCorrelation(), this::canonicalize); - return new ApplyNode(node.getId(), source, subquery, canonicalize(node.getSubqueryAssignments()), canonicalCorrelation, node.getOriginSubquery()); + return new ApplyNode(node.getId(), source, subquery, canonicalize(node.getSubqueryAssignments()), canonicalCorrelation, node.getOriginSubqueryError()); } @Override @@ -465,7 +465,7 @@ public PlanNode visitLateralJoin(LateralJoinNode node, RewriteContext cont PlanNode subquery = context.rewrite(node.getSubquery()); List canonicalCorrelation = canonicalizeAndDistinct(node.getCorrelation()); - return new LateralJoinNode(node.getId(), source, subquery, canonicalCorrelation, node.getType(), node.getOriginSubquery()); + return new LateralJoinNode(node.getId(), source, subquery, canonicalCorrelation, node.getType(), node.getOriginSubqueryError()); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java index 6238f9d25b84d..d416da954c347 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.Range; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java index 03975efd84800..356ac9bcd124c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java @@ -17,7 +17,6 @@ import com.facebook.presto.sql.tree.ExistsPredicate; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.InPredicate; -import com.facebook.presto.sql.tree.Node; import com.facebook.presto.sql.tree.QuantifiedComparisonExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -64,10 +63,9 @@ public class ApplyNode private final Assignments subqueryAssignments; /** - * HACK! - * Used for error reporting in case this ApplyNode is not supported + * This information is only used for sanity check. */ - private final Node originSubquery; + private final String originSubqueryError; @JsonCreator public ApplyNode( @@ -76,14 +74,14 @@ public ApplyNode( @JsonProperty("subquery") PlanNode subquery, @JsonProperty("subqueryAssignments") Assignments subqueryAssignments, @JsonProperty("correlation") List correlation, - @JsonProperty("originSubquery") Node originSubquery) + @JsonProperty("originSubqueryError") String originSubqueryError) { super(id); requireNonNull(input, "input is null"); requireNonNull(subquery, "right is null"); requireNonNull(subqueryAssignments, "assignments is null"); requireNonNull(correlation, "correlation is null"); - requireNonNull(originSubquery, "originSubquery is null"); + requireNonNull(originSubqueryError, "originSubqueryError is null"); checkArgument(input.getOutputSymbols().containsAll(correlation), "Input does not contain symbols from correlation"); checkArgument( @@ -94,7 +92,7 @@ public ApplyNode( this.subquery = subquery; this.subqueryAssignments = subqueryAssignments; this.correlation = ImmutableList.copyOf(correlation); - this.originSubquery = originSubquery; + this.originSubqueryError = originSubqueryError; } private static boolean isSupportedSubqueryExpression(Expression expression) @@ -128,10 +126,10 @@ public List getCorrelation() return correlation; } - @JsonProperty("originSubquery") - public Node getOriginSubquery() + @JsonProperty("originSubqueryError") + public String getOriginSubqueryError() { - return originSubquery; + return originSubqueryError; } @Override @@ -160,6 +158,6 @@ public R accept(PlanVisitor visitor, C context) public PlanNode replaceChildren(List newChildren) { checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); - return new ApplyNode(getId(), newChildren.get(0), newChildren.get(1), subqueryAssignments, correlation, originSubquery); + return new ApplyNode(getId(), newChildren.get(0), newChildren.get(1), subqueryAssignments, correlation, originSubqueryError); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java index d192e871c8498..550f9788aee2e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java @@ -120,7 +120,7 @@ public ExchangeNode( "local exchanges do not support constant partition function arguments"); checkArgument(!scope.isRemote() || type == REPARTITION || !partitioningScheme.isReplicateNullsAndAny(), "Only REPARTITION can replicate remotely"); - checkArgument(scope != REMOTE_MATERIALIZED || type != REPARTITION, "Only REPARTITION can be REMOTE_MATERIALIZED: %s", type); + checkArgument(scope != REMOTE_MATERIALIZED || type == REPARTITION, "Only REPARTITION can be REMOTE_MATERIALIZED: %s", type); orderingScheme.ifPresent(ordering -> { PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LateralJoinNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LateralJoinNode.java index 4a8d1e92e1008..f7bbd6828e71d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LateralJoinNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LateralJoinNode.java @@ -14,7 +14,6 @@ package com.facebook.presto.sql.planner.plan; import com.facebook.presto.sql.planner.Symbol; -import com.facebook.presto.sql.tree.Node; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -65,10 +64,9 @@ public JoinNode.Type toJoinNodeType() private final Type type; /** - * HACK! - * Used for error reporting in case this ApplyNode is not supported + * This information is only used for sanity check. */ - private final Node originSubquery; + private final String originSubqueryError; @JsonCreator public LateralJoinNode( @@ -77,13 +75,13 @@ public LateralJoinNode( @JsonProperty("subquery") PlanNode subquery, @JsonProperty("correlation") List correlation, @JsonProperty("type") Type type, - @JsonProperty("originSubquery") Node originSubquery) + @JsonProperty("originSubqueryError") String originSubqueryError) { super(id); requireNonNull(input, "input is null"); requireNonNull(subquery, "right is null"); requireNonNull(correlation, "correlation is null"); - requireNonNull(originSubquery, "originSubquery is null"); + requireNonNull(originSubqueryError, "originSubqueryError is null"); checkArgument(input.getOutputSymbols().containsAll(correlation), "Input does not contain symbols from correlation"); @@ -91,7 +89,7 @@ public LateralJoinNode( this.subquery = subquery; this.correlation = ImmutableList.copyOf(correlation); this.type = type; - this.originSubquery = originSubquery; + this.originSubqueryError = originSubqueryError; } @JsonProperty("input") @@ -118,10 +116,10 @@ public Type getType() return type; } - @JsonProperty("originSubquery") - public Node getOriginSubquery() + @JsonProperty("originSubqueryError") + public String getOriginSubqueryError() { - return originSubquery; + return originSubqueryError; } @Override @@ -144,7 +142,7 @@ public List getOutputSymbols() public PlanNode replaceChildren(List newChildren) { checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); - return new LateralJoinNode(getId(), newChildren.get(0), newChildren.get(1), correlation, type, originSubquery); + return new LateralJoinNode(getId(), newChildren.get(0), newChildren.get(1), correlation, type, originSubqueryError); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableScanNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableScanNode.java index 5e178e6e94fd2..5158d04701dae 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableScanNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableScanNode.java @@ -50,13 +50,16 @@ public class TableScanNode private final TupleDomain enforcedConstraint; + private final boolean temporaryTable; + @JsonCreator public TableScanNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("table") TableHandle table, @JsonProperty("outputSymbols") List outputs, @JsonProperty("assignments") Map assignments, - @JsonProperty("layout") Optional tableLayout) + @JsonProperty("layout") Optional tableLayout, + @JsonProperty("temporaryTable") boolean temporaryTable) { // This constructor is for JSON deserialization only. Do not use. super(id); @@ -65,6 +68,7 @@ public TableScanNode( this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null")); checkArgument(assignments.keySet().containsAll(outputs), "assignments does not cover all of outputs"); this.tableLayout = requireNonNull(tableLayout, "tableLayout is null"); + this.temporaryTable = temporaryTable; this.currentConstraint = null; this.enforcedConstraint = null; } @@ -75,7 +79,7 @@ public TableScanNode( List outputs, Map assignments) { - this(id, table, outputs, assignments, Optional.empty(), TupleDomain.all(), TupleDomain.all()); + this(id, table, outputs, assignments, Optional.empty(), TupleDomain.all(), TupleDomain.all(), false); } public TableScanNode( @@ -86,6 +90,19 @@ public TableScanNode( Optional tableLayout, TupleDomain currentConstraint, TupleDomain enforcedConstraint) + { + this(id, table, outputs, assignments, tableLayout, currentConstraint, enforcedConstraint, false); + } + + public TableScanNode( + PlanNodeId id, + TableHandle table, + List outputs, + Map assignments, + Optional tableLayout, + TupleDomain currentConstraint, + TupleDomain enforcedConstraint, + boolean temporaryTable) { super(id); this.table = requireNonNull(table, "table is null"); @@ -95,6 +112,7 @@ public TableScanNode( this.tableLayout = requireNonNull(tableLayout, "tableLayout is null"); this.currentConstraint = requireNonNull(currentConstraint, "currentConstraint is null"); this.enforcedConstraint = requireNonNull(enforcedConstraint, "enforcedConstraint is null"); + this.temporaryTable = temporaryTable; if (!currentConstraint.isAll() || !enforcedConstraint.isAll()) { checkArgument(tableLayout.isPresent(), "tableLayout must be present when currentConstraint or enforcedConstraint is non-trivial"); } @@ -125,6 +143,12 @@ public Map getAssignments() return assignments; } + @JsonProperty + public boolean isTemporaryTable() + { + return temporaryTable; + } + /** * A TupleDomain that represents a predicate that every row this TableScan node * produces is guaranteed to satisfy. diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java new file mode 100644 index 0000000000000..d8cb34c2b07af --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.RowExpressionService; + +import static java.util.Objects.requireNonNull; + +public final class ConnectorRowExpressionService + implements RowExpressionService +{ + private final DomainTranslator domainTranslator; + + public ConnectorRowExpressionService(DomainTranslator domainTranslator) + { + this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); + } + + @Override + public DomainTranslator getDomainTranslator() + { + return domainTranslator; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java similarity index 60% rename from presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java rename to presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java index 9d4f92b1d32c6..e87a863188cb0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java @@ -16,16 +16,18 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ComparisonExpression; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.List; +import java.util.Optional; import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; import static com.facebook.presto.spi.function.OperatorType.ADD; +import static com.facebook.presto.spi.function.OperatorType.BETWEEN; import static com.facebook.presto.spi.function.OperatorType.DIVIDE; import static com.facebook.presto.spi.function.OperatorType.EQUAL; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; @@ -37,6 +39,7 @@ import static com.facebook.presto.spi.function.OperatorType.MULTIPLY; import static com.facebook.presto.spi.function.OperatorType.NEGATION; import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; import static com.facebook.presto.spi.function.OperatorType.SUBTRACT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; @@ -44,20 +47,23 @@ import static com.facebook.presto.sql.tree.ArrayConstructor.ARRAY_CONSTRUCTOR; import static com.facebook.presto.type.LikePatternType.LIKE_PATTERN; import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; -public final class StandardFunctionResolution +public final class FunctionResolution + implements StandardFunctionResolution { private final FunctionManager functionManager; - public StandardFunctionResolution(FunctionManager functionManager) + public FunctionResolution(FunctionManager functionManager) { this.functionManager = requireNonNull(functionManager, "functionManager is null"); } + @Override public FunctionHandle notFunction() { - return functionManager.lookupFunction(QualifiedName.of("not"), fromTypes(BOOLEAN)); + return functionManager.lookupFunction("not", fromTypes(BOOLEAN)); } public boolean isNotFunction(FunctionHandle functionHandle) @@ -65,15 +71,17 @@ public boolean isNotFunction(FunctionHandle functionHandle) return notFunction().equals(functionHandle); } + @Override public FunctionHandle likeVarcharFunction() { - return functionManager.lookupFunction(QualifiedName.of("LIKE"), fromTypes(VARCHAR, LIKE_PATTERN)); + return functionManager.lookupFunction("LIKE", fromTypes(VARCHAR, LIKE_PATTERN)); } + @Override public FunctionHandle likeCharFunction(Type valueType) { checkArgument(valueType instanceof CharType, "Expected CHAR value type"); - return functionManager.lookupFunction(QualifiedName.of("LIKE"), fromTypes(valueType, LIKE_PATTERN)); + return functionManager.lookupFunction("LIKE", fromTypes(valueType, LIKE_PATTERN)); } public boolean isLikeFunction(FunctionHandle functionHandle) @@ -83,17 +91,32 @@ public boolean isLikeFunction(FunctionHandle functionHandle) public FunctionHandle likePatternFunction() { - return functionManager.lookupFunction(QualifiedName.of("LIKE_PATTERN"), fromTypes(VARCHAR, VARCHAR)); + return functionManager.lookupFunction("LIKE_PATTERN", fromTypes(VARCHAR, VARCHAR)); } + @Override public boolean isCastFunction(FunctionHandle functionHandle) { return functionManager.getFunctionMetadata(functionHandle).getName().equals(mangleOperatorName(OperatorType.CAST.name())); } + @Override + public FunctionHandle betweenFunction(Type valueType, Type lowerBoundType, Type upperBoundType) + { + return functionManager.lookupFunction(BETWEEN.getFunctionName(), fromTypes(valueType, lowerBoundType, upperBoundType)); + } + + @Override public boolean isBetweenFunction(FunctionHandle functionHandle) { - return functionHandle.getSignature().getName().equals(mangleOperatorName(OperatorType.BETWEEN.name())); + return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(BETWEEN)); + } + + @Override + public FunctionHandle arithmeticFunction(OperatorType operator, Type leftType, Type rightType) + { + checkArgument(operator.isArithmeticOperator(), format("unexpected arithmetic type %s", operator)); + return functionManager.resolveOperator(operator, fromTypes(leftType, rightType)); } public FunctionHandle arithmeticFunction(ArithmeticBinaryExpression.Operator operator, Type leftType, Type rightType) @@ -118,17 +141,39 @@ public FunctionHandle arithmeticFunction(ArithmeticBinaryExpression.Operator ope default: throw new IllegalStateException("Unknown arithmetic operator: " + operator); } - return functionManager.resolveOperator(operatorType, fromTypes(leftType, rightType)); + return arithmeticFunction(operatorType, leftType, rightType); + } + + @Override + public boolean isArithmeticFunction(FunctionHandle functionHandle) + { + Optional operatorType = functionManager.getFunctionMetadata(functionHandle).getOperatorType(); + return operatorType.isPresent() && operatorType.get().isArithmeticOperator(); } + @Override + public FunctionHandle negateFunction(Type type) + { + return functionManager.lookupFunction(NEGATION.getFunctionName(), fromTypes(type)); + } + + @Override public boolean isNegateFunction(FunctionHandle functionHandle) { - return functionHandle.getSignature().getName().equals(mangleOperatorName(NEGATION.name())); + return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(NEGATION)); } + @Override public FunctionHandle arrayConstructor(List argumentTypes) { - return functionManager.lookupFunction(QualifiedName.of(ARRAY_CONSTRUCTOR), fromTypes(argumentTypes)); + return functionManager.lookupFunction(ARRAY_CONSTRUCTOR, fromTypes(argumentTypes)); + } + + @Override + public FunctionHandle comparisonFunction(OperatorType operator, Type leftType, Type rightType) + { + checkArgument(operator.isComparisonOperator(), format("unexpected comparison type %s", operator)); + return functionManager.resolveOperator(operator, fromTypes(leftType, rightType)); } public FunctionHandle comparisonFunction(ComparisonExpression.Operator operator, Type leftType, Type rightType) @@ -160,11 +205,35 @@ public FunctionHandle comparisonFunction(ComparisonExpression.Operator operator, throw new IllegalStateException("Unsupported comparison operator type: " + operator); } - return functionManager.resolveOperator(operatorType, fromTypes(leftType, rightType)); + return comparisonFunction(operatorType, leftType, rightType); + } + + @Override + public boolean isComparisonFunction(FunctionHandle functionHandle) + { + Optional operatorType = functionManager.getFunctionMetadata(functionHandle).getOperatorType(); + return operatorType.isPresent() && operatorType.get().isComparisonOperator(); + } + + @Override + public FunctionHandle subscriptFunction(Type baseType, Type indexType) + { + return functionManager.lookupFunction(SUBSCRIPT.getFunctionName(), fromTypes(baseType, indexType)); + } + + @Override + public boolean isSubscriptFunction(FunctionHandle functionHandle) + { + return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(SUBSCRIPT)); } public FunctionHandle tryFunction(Type returnType) { - return functionManager.lookupFunction(QualifiedName.of("TRY"), fromTypes(returnType)); + return functionManager.lookupFunction("TRY", fromTypes(returnType)); + } + + public boolean isTryFunction(FunctionHandle functionHandle) + { + return functionManager.getFunctionMetadata(functionHandle).getName().equals("TRY"); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java index 040c1af2d2ba3..8981df9693d8b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java @@ -13,14 +13,15 @@ */ package com.facebook.presto.sql.relational; -import com.facebook.presto.Session; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.OperatorNotFoundException; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.predicate.DiscreteValues; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.Marker; @@ -33,6 +34,7 @@ import com.facebook.presto.spi.predicate.ValueSet; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -48,6 +50,7 @@ import com.google.common.collect.PeekingIterator; import javax.annotation.Nullable; +import javax.inject.Inject; import java.util.ArrayList; import java.util.List; @@ -87,18 +90,23 @@ import static java.util.stream.Collectors.toList; public final class RowExpressionDomainTranslator + implements DomainTranslator { private final FunctionManager functionManager; private final LogicalRowExpressions logicalRowExpressions; private final StandardFunctionResolution functionResolution; + private final Metadata metadata; - public RowExpressionDomainTranslator(FunctionManager functionManager) + @Inject + public RowExpressionDomainTranslator(Metadata metadata) { - this.functionManager = requireNonNull(functionManager, "functionManager is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.functionManager = metadata.getFunctionManager(); this.logicalRowExpressions = new LogicalRowExpressions(functionManager); - this.functionResolution = new StandardFunctionResolution(functionManager); + this.functionResolution = new FunctionResolution(functionManager); } + @Override public RowExpression toPredicate(TupleDomain tupleDomain) { if (tupleDomain.isNone()) { @@ -112,6 +120,12 @@ public RowExpression toPredicate(TupleDomain tupleD .collect(collectingAndThen(toImmutableList(), logicalRowExpressions::combineConjuncts)); } + @Override + public ExtractionResult fromPredicate(ConnectorSession session, RowExpression predicate, ColumnExtractor columnExtractor) + { + return predicate.accept(new Visitor<>(metadata, session, columnExtractor), false); + } + private RowExpression toPredicate(Domain domain, VariableReferenceExpression reference) { if (domain.getValues().isNone()) { @@ -276,29 +290,19 @@ private static boolean isBetween(Range range) && !range.getHigh().isUpperUnbounded() && range.getHigh().getBound() == Marker.Bound.EXACTLY; } - /** - * Convert an Expression predicate into an ExtractionResult consisting of: - * 1) A successfully extracted TupleDomain - * 2) An Expression fragment which represents the part of the original Expression that will need to be re-evaluated - * after filtering with the TupleDomain. - */ - public static ExtractionResult fromPredicate(Metadata metadata, Session session, RowExpression predicate) - { - return predicate.accept(new Visitor(metadata, session), false); - } - - private static class Visitor - implements RowExpressionVisitor + private static class Visitor + implements RowExpressionVisitor, Boolean> { private final InterpretedFunctionInvoker functionInvoker; private final Metadata metadata; - private final Session session; + private final ConnectorSession session; private final FunctionManager functionManager; private final LogicalRowExpressions logicalRowExpressions; private final DeterminismEvaluator determinismEvaluator; private final StandardFunctionResolution resolution; + private final ColumnExtractor columnExtractor; - private Visitor(Metadata metadata, Session session) + private Visitor(Metadata metadata, ConnectorSession session, ColumnExtractor columnExtractor) { this.functionInvoker = new InterpretedFunctionInvoker(metadata.getFunctionManager()); this.metadata = metadata; @@ -306,11 +310,12 @@ private Visitor(Metadata metadata, Session session) this.functionManager = metadata.getFunctionManager(); this.logicalRowExpressions = new LogicalRowExpressions(functionManager); this.determinismEvaluator = new DeterminismEvaluator(functionManager); - this.resolution = new StandardFunctionResolution(functionManager); + this.resolution = new FunctionResolution(functionManager); + this.columnExtractor = requireNonNull(columnExtractor, "columnExtractor is null"); } @Override - public ExtractionResult visitSpecialForm(SpecialFormExpression node, Boolean complement) + public ExtractionResult visitSpecialForm(SpecialFormExpression node, Boolean complement) { switch (node.getForm()) { case AND: @@ -329,23 +334,24 @@ public ExtractionResult visitSpecialForm(SpecialFormExpression node, Boolean com ExtractionResult extractionResult = or(disjuncts.build()).accept(this, complement); // preserve original IN predicate as remaining predicate - if (extractionResult.tupleDomain.isAll()) { + if (extractionResult.getTupleDomain().isAll()) { RowExpression originalPredicate = node; if (complement) { originalPredicate = not(resolution, originalPredicate); } - return new ExtractionResult(extractionResult.tupleDomain, originalPredicate); + return new ExtractionResult<>(extractionResult.getTupleDomain(), originalPredicate); } return extractionResult; } case IS_NULL: { RowExpression value = node.getArguments().get(0); - if (!(value instanceof VariableReferenceExpression)) { + Optional column = columnExtractor.extract(value); + if (!column.isPresent()) { return visitRowExpression(node, complement); } Domain domain = complementIfNecessary(Domain.onlyNull(value.getType()), complement); - return new ExtractionResult(TupleDomain.withColumnDomains(ImmutableMap.of((VariableReferenceExpression) value, domain)), TRUE); + return new ExtractionResult<>(TupleDomain.withColumnDomains(ImmutableMap.of(column.get(), domain)), TRUE); } default: return visitRowExpression(node, complement); @@ -353,32 +359,32 @@ public ExtractionResult visitSpecialForm(SpecialFormExpression node, Boolean com } @Override - public ExtractionResult visitConstant(ConstantExpression node, Boolean complement) + public ExtractionResult visitConstant(ConstantExpression node, Boolean complement) { if (node.getValue() == null) { - return new ExtractionResult(TupleDomain.none(), TRUE); + return new ExtractionResult<>(TupleDomain.none(), TRUE); } if (node.getType() == BOOLEAN) { boolean value = complement != (boolean) node.getValue(); - return new ExtractionResult(value ? TupleDomain.all() : TupleDomain.none(), TRUE); + return new ExtractionResult<>(value ? TupleDomain.all() : TupleDomain.none(), TRUE); } throw new IllegalStateException("Can not extract predicate from constant type: " + node.getType()); } @Override - public ExtractionResult visitLambda(LambdaDefinitionExpression node, Boolean complement) + public ExtractionResult visitLambda(LambdaDefinitionExpression node, Boolean complement) { return visitRowExpression(node, complement); } @Override - public ExtractionResult visitVariableReference(VariableReferenceExpression node, Boolean complement) + public ExtractionResult visitVariableReference(VariableReferenceExpression node, Boolean complement) { return visitRowExpression(node, complement); } @Override - public ExtractionResult visitCall(CallExpression node, Boolean complement) + public ExtractionResult visitCall(CallExpression node, Boolean complement) { if (node.getFunctionHandle().equals(resolution.notFunction())) { return node.getArguments().get(0).accept(this, !complement); @@ -400,10 +406,11 @@ public ExtractionResult visitCall(CallExpression node, Boolean complement) NormalizedSimpleComparison normalized = optionalNormalized.get(); RowExpression expression = normalized.getExpression(); - if (expression instanceof VariableReferenceExpression) { + Optional column = columnExtractor.extract(expression); + if (column.isPresent()) { NullableValue value = normalized.getValue(); Type type = value.getType(); // common type for symbol and value - return createComparisonExtractionResult(normalized.getComparisonOperator(), (VariableReferenceExpression) expression, type, value.getValue(), complement); + return createComparisonExtractionResult(normalized.getComparisonOperator(), column.get(), type, value.getValue(), complement); } else if (expression instanceof CallExpression && resolution.isCastFunction(((CallExpression) expression).getFunctionHandle())) { CallExpression castExpression = (CallExpression) expression; @@ -451,7 +458,7 @@ else if (expression instanceof CallExpression && resolution.isCastFunction(((Cal } @Override - public ExtractionResult visitInputReference(InputReferenceExpression node, Boolean complement) + public ExtractionResult visitInputReference(InputReferenceExpression node, Boolean complement) { return visitRowExpression(node, complement); } @@ -547,7 +554,7 @@ private RowExpression binaryOperator(OperatorType operatorType, RowExpression le private Optional floorValue(Type fromType, Type toType, Object value) { return getSaturatedFloorCastOperator(fromType, toType) - .map((operator) -> functionInvoker.invoke(operator, session.toConnectorSession(), value)); + .map((operator) -> functionInvoker.invoke(operator, session, value)); } private Optional getSaturatedFloorCastOperator(Type fromType, Type toType) @@ -563,7 +570,7 @@ private Optional getSaturatedFloorCastOperator(Type fromType, Ty private int compareOriginalValueToCoerced(Type originalValueType, Object originalValue, Type coercedValueType, Object coercedValue) { FunctionHandle castToOriginalTypeOperator = metadata.getFunctionManager().lookupCast(CAST, coercedValueType.getTypeSignature(), originalValueType.getTypeSignature()); - Object coercedValueInOriginalType = functionInvoker.invoke(castToOriginalTypeOperator, session.toConnectorSession(), coercedValue); + Object coercedValueInOriginalType = functionInvoker.invoke(castToOriginalTypeOperator, session, coercedValue); Block originalValueBlock = Utils.nativeValueToBlock(originalValueType, originalValue); Block coercedValueBlock = Utils.nativeValueToBlock(originalValueType, coercedValueInOriginalType); return originalValueType.compareTo(originalValueBlock, 0, coercedValueBlock, 0); @@ -659,7 +666,7 @@ private Optional toNormalizedSimpleComparison(Operat return Optional.of(new NormalizedSimpleComparison(expression, comparisonOperator, value)); } - private static ExtractionResult createComparisonExtractionResult(OperatorType comparisonOperator, VariableReferenceExpression column, Type type, @Nullable Object value, boolean complement) + private static ExtractionResult createComparisonExtractionResult(OperatorType comparisonOperator, T column, Type type, @Nullable Object value, boolean complement) { if (value == null) { switch (comparisonOperator) { @@ -669,11 +676,11 @@ private static ExtractionResult createComparisonExtractionResult(OperatorType co case LESS_THAN: case LESS_THAN_OR_EQUAL: case NOT_EQUAL: - return new ExtractionResult(TupleDomain.none(), TRUE); + return new ExtractionResult<>(TupleDomain.none(), TRUE); case IS_DISTINCT_FROM: Domain domain = complementIfNecessary(Domain.notNull(type), complement); - return new ExtractionResult( + return new ExtractionResult<>( TupleDomain.withColumnDomains(ImmutableMap.of(column, domain)), TRUE); @@ -693,7 +700,7 @@ else if (type.isComparable()) { throw new AssertionError("Type cannot be used in a comparison expression (should have been caught in analysis): " + type); } - return new ExtractionResult(TupleDomain.withColumnDomains(ImmutableMap.of(column, domain)), TRUE); + return new ExtractionResult<>(TupleDomain.withColumnDomains(ImmutableMap.of(column, domain)), TRUE); } private static OperatorType flip(OperatorType operatorType) @@ -733,19 +740,19 @@ private RowExpression complementIfNecessary(RowExpression expression, boolean co return complement ? not(resolution, expression) : expression; } - private ExtractionResult visitRowExpression(RowExpression node, Boolean complement) + private ExtractionResult visitRowExpression(RowExpression node, Boolean complement) { // If we don't know how to process this node, the default response is to say that the TupleDomain is "all" - return new ExtractionResult(TupleDomain.all(), complementIfNecessary(node, complement)); + return new ExtractionResult<>(TupleDomain.all(), complementIfNecessary(node, complement)); } - private ExtractionResult visitBinaryLogic(SpecialFormExpression node, Boolean complement) + private ExtractionResult visitBinaryLogic(SpecialFormExpression node, Boolean complement) { - ExtractionResult leftResult = node.getArguments().get(0).accept(this, complement); - ExtractionResult rightResult = node.getArguments().get(1).accept(this, complement); + ExtractionResult leftResult = node.getArguments().get(0).accept(this, complement); + ExtractionResult rightResult = node.getArguments().get(1).accept(this, complement); - TupleDomain leftTupleDomain = leftResult.getTupleDomain(); - TupleDomain rightTupleDomain = rightResult.getTupleDomain(); + TupleDomain leftTupleDomain = leftResult.getTupleDomain(); + TupleDomain rightTupleDomain = rightResult.getTupleDomain(); Form operator = node.getForm(); if (complement) { @@ -762,12 +769,12 @@ else if (operator == OR) { switch (operator) { case AND: { - return new ExtractionResult( + return new ExtractionResult<>( leftTupleDomain.intersect(rightTupleDomain), logicalRowExpressions.combineConjuncts(leftResult.getRemainingExpression(), rightResult.getRemainingExpression())); } case OR: { - TupleDomain columnUnionedTupleDomain = TupleDomain.columnWiseUnion(leftTupleDomain, rightTupleDomain); + TupleDomain columnUnionedTupleDomain = TupleDomain.columnWiseUnion(leftTupleDomain, rightTupleDomain); // In most cases, the columnUnionedTupleDomain is only a superset of the actual strict union // and so we can return the current node as the remainingExpression so that all bounds will be double checked again at execution time. @@ -794,7 +801,7 @@ else if (operator == OR) { } } - return new ExtractionResult(columnUnionedTupleDomain, remainingExpression); + return new ExtractionResult<>(columnUnionedTupleDomain, remainingExpression); } default: throw new IllegalStateException("Can not extract predicate from special form: " + node.getForm()); @@ -880,26 +887,4 @@ public NullableValue getValue() return value; } } - - public static class ExtractionResult - { - private final TupleDomain tupleDomain; - private final RowExpression remainingExpression; - - public ExtractionResult(TupleDomain tupleDomain, RowExpression remainingExpression) - { - this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); - this.remainingExpression = requireNonNull(remainingExpression, "remainingExpression is null"); - } - - public TupleDomain getTupleDomain() - { - return tupleDomain; - } - - public RowExpression getRemainingExpression() - { - return remainingExpression; - } - } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionNodeInliner.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionNodeInliner.java new file mode 100644 index 0000000000000..2d9c90f8264d2 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionNodeInliner.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.spi.relation.RowExpression; + +import java.util.Map; + +public class RowExpressionNodeInliner + extends RowExpressionRewriter +{ + private final Map mappings; + + public static RowExpression replaceExpression(RowExpression expression, Map mappings) + { + return RowExpressionTreeRewriter.rewriteWith(new RowExpressionNodeInliner(mappings), expression); + } + + public RowExpressionNodeInliner(Map mappings) + { + this.mappings = mappings; + } + + @Override + public RowExpression rewriteRowExpression(RowExpression node, Void context, RowExpressionTreeRewriter treeRewriter) + { + return mappings.get(node); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionRewriter.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionRewriter.java new file mode 100644 index 0000000000000..00b969f3fdba9 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionRewriter.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; + +public class RowExpressionRewriter +{ + public RowExpression rewriteRowExpression(RowExpression node, C context, RowExpressionTreeRewriter treeRewriter) + { + return null; + } + + public RowExpression rewriteInputReference(InputReferenceExpression node, C context, RowExpressionTreeRewriter treeRewriter) + { + return rewriteRowExpression(node, context, treeRewriter); + } + + public RowExpression rewriteCall(CallExpression node, C context, RowExpressionTreeRewriter treeRewriter) + { + return rewriteRowExpression(node, context, treeRewriter); + } + + public RowExpression rewriteConstant(ConstantExpression node, C context, RowExpressionTreeRewriter treeRewriter) + { + return rewriteRowExpression(node, context, treeRewriter); + } + + public RowExpression rewriteLambda(LambdaDefinitionExpression node, C context, RowExpressionTreeRewriter treeRewriter) + { + return rewriteRowExpression(node, context, treeRewriter); + } + + public RowExpression rewriteVariableReference(VariableReferenceExpression node, C context, RowExpressionTreeRewriter treeRewriter) + { + return rewriteRowExpression(node, context, treeRewriter); + } + + public RowExpression rewriteSpecialForm(SpecialFormExpression node, C context, RowExpressionTreeRewriter treeRewriter) + { + return rewriteRowExpression(node, context, treeRewriter); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionTreeRewriter.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionTreeRewriter.java new file mode 100644 index 0000000000000..6eceef1cfb52e --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionTreeRewriter.java @@ -0,0 +1,214 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +import java.util.Iterator; +import java.util.List; + +import static com.facebook.presto.sql.relational.Expressions.call; + +public final class RowExpressionTreeRewriter +{ + private final RowExpressionRewriter rewriter; + private final RowExpressionVisitor> visitor; + + public static T rewriteWith(RowExpressionRewriter rewriter, T node) + { + return new RowExpressionTreeRewriter<>(rewriter).rewrite(node, null); + } + + public static T rewriteWith(RowExpressionRewriter rewriter, T node, C context) + { + return new RowExpressionTreeRewriter<>(rewriter).rewrite(node, context); + } + + public RowExpressionTreeRewriter(RowExpressionRewriter rewriter) + { + this.rewriter = rewriter; + this.visitor = new RewritingVisitor(); + } + + private List rewrite(List items, Context context) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (RowExpression expression : items) { + builder.add(rewrite(expression, context.get())); + } + return builder.build(); + } + + @SuppressWarnings("unchecked") + public T rewrite(T node, C context) + { + return (T) node.accept(visitor, new Context<>(context, false)); + } + + /** + * Invoke the default rewrite logic explicitly. Specifically, it skips the invocation of the expression rewriter for the provided node. + */ + @SuppressWarnings("unchecked") + public T defaultRewrite(T node, C context) + { + return (T) node.accept(visitor, new Context<>(context, true)); + } + + private class RewritingVisitor + implements RowExpressionVisitor> + { + @Override + public RowExpression visitInputReference(InputReferenceExpression input, Context context) + { + if (!context.isDefaultRewrite()) { + RowExpression result = rewriter.rewriteInputReference(input, context.get(), RowExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + return input; + } + + @Override + public RowExpression visitCall(CallExpression call, Context context) + { + if (!context.isDefaultRewrite()) { + RowExpression result = rewriter.rewriteCall(call, context.get(), RowExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + List arguments = rewrite(call.getArguments(), context); + + if (!sameElements(call.getArguments(), arguments)) { + return call(call.getDisplayName(), call.getFunctionHandle(), call.getType(), arguments); + } + return call; + } + + @Override + public RowExpression visitConstant(ConstantExpression literal, Context context) + { + if (!context.isDefaultRewrite()) { + RowExpression result = rewriter.rewriteConstant(literal, context.get(), RowExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + return literal; + } + + @Override + public RowExpression visitLambda(LambdaDefinitionExpression lambda, Context context) + { + if (!context.isDefaultRewrite()) { + RowExpression result = rewriter.rewriteLambda(lambda, context.get(), RowExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + RowExpression body = rewrite(lambda.getBody(), context.get()); + if (body != lambda.getBody()) { + return new LambdaDefinitionExpression(lambda.getArgumentTypes(), lambda.getArguments(), body); + } + + return lambda; + } + + @Override + public RowExpression visitVariableReference(VariableReferenceExpression variable, Context context) + { + if (!context.isDefaultRewrite()) { + RowExpression result = rewriter.rewriteVariableReference(variable, context.get(), RowExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + return variable; + } + + @Override + public RowExpression visitSpecialForm(SpecialFormExpression specialForm, Context context) + { + if (!context.isDefaultRewrite()) { + RowExpression result = rewriter.rewriteSpecialForm(specialForm, context.get(), RowExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + List arguments = rewrite(specialForm.getArguments(), context); + + if (!sameElements(specialForm.getArguments(), arguments)) { + return new SpecialFormExpression(specialForm.getForm(), specialForm.getType(), arguments); + } + return specialForm; + } + } + + public static class Context + { + private final boolean defaultRewrite; + private final C context; + + private Context(C context, boolean defaultRewrite) + { + this.context = context; + this.defaultRewrite = defaultRewrite; + } + + public C get() + { + return context; + } + + public boolean isDefaultRewrite() + { + return defaultRewrite; + } + } + + @SuppressWarnings("ObjectEquality") + private static boolean sameElements(Iterable a, Iterable b) + { + if (Iterables.size(a) != Iterables.size(b)) { + return false; + } + + Iterator first = a.iterator(); + Iterator second = b.iterator(); + + while (first.hasNext() && second.hasNext()) { + if (first.next() != second.next()) { + return false; + } + } + + return true; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java index fc5b40919c4a0..bf3860a4d3942 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java @@ -72,7 +72,6 @@ import com.facebook.presto.sql.tree.NotExpression; import com.facebook.presto.sql.tree.NullIfExpression; import com.facebook.presto.sql.tree.NullLiteral; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Row; import com.facebook.presto.sql.tree.SearchedCaseExpression; import com.facebook.presto.sql.tree.SimpleCaseExpression; @@ -167,7 +166,7 @@ public static RowExpression translate( requireNonNull(result, "translated expression is null"); if (optimize) { - ExpressionOptimizer optimizer = new ExpressionOptimizer(functionManager, session); + ExpressionOptimizer optimizer = new ExpressionOptimizer(functionManager, session.toConnectorSession()); return optimizer.optimize(result); } @@ -182,7 +181,7 @@ private static class Visitor private final TypeManager typeManager; private final FunctionManager functionManager; private final Session session; - private final StandardFunctionResolution standardFunctionResolution; + private final FunctionResolution functionResolution; private Visitor( Map, Type> types, @@ -196,7 +195,7 @@ private Visitor( this.typeManager = typeManager; this.functionManager = functionManager; this.session = session; - this.standardFunctionResolution = new StandardFunctionResolution(functionManager); + this.functionResolution = new FunctionResolution(functionManager); } private Type getType(Expression node) @@ -301,7 +300,7 @@ protected RowExpression visitGenericLiteral(GenericLiteral node, Void context) if (JSON.equals(type)) { return call( "json_parse", - functionManager.lookupFunction(QualifiedName.of("json_parse"), fromTypes(VARCHAR)), + functionManager.lookupFunction("json_parse", fromTypes(VARCHAR)), getType(node), constant(utf8Slice(node.getValue()), VARCHAR)); } @@ -366,7 +365,7 @@ protected RowExpression visitComparisonExpression(ComparisonExpression node, Voi return call( node.getOperator().name(), - standardFunctionResolution.comparisonFunction(node.getOperator(), left.getType(), right.getType()), + functionResolution.comparisonFunction(node.getOperator(), left.getType(), right.getType()), BOOLEAN, left, right); @@ -439,7 +438,7 @@ protected RowExpression visitArithmeticBinary(ArithmeticBinaryExpression node, V return call( node.getOperator().name(), - standardFunctionResolution.arithmeticFunction(node.getOperator(), left.getType(), right.getType()), + functionResolution.arithmeticFunction(node.getOperator(), left.getType(), right.getType()), getType(node), left, right); @@ -671,7 +670,7 @@ protected RowExpression visitIfExpression(IfExpression node, Void context) @Override protected RowExpression visitTryExpression(TryExpression node, Void context) { - return call("TRY", standardFunctionResolution.tryFunction(getType(node)), getType(node), process(node.getInnerExpression(), context)); + return call("TRY", functionResolution.tryFunction(getType(node)), getType(node), process(node.getInnerExpression(), context)); } @Override @@ -694,7 +693,7 @@ protected RowExpression visitIsNotNullPredicate(IsNotNullPredicate node, Void co return call( "not", - standardFunctionResolution.notFunction(), + functionResolution.notFunction(), BOOLEAN, specialForm(IS_NULL, BOOLEAN, ImmutableList.of(expression))); } @@ -710,7 +709,7 @@ protected RowExpression visitIsNullPredicate(IsNullPredicate node, Void context) @Override protected RowExpression visitNotExpression(NotExpression node, Void context) { - return call("not", standardFunctionResolution.notFunction(), BOOLEAN, process(node.getValue(), context)); + return call("not", functionResolution.notFunction(), BOOLEAN, process(node.getValue(), context)); } @Override @@ -746,7 +745,7 @@ protected RowExpression visitLikePredicate(LikePredicate node, Void context) if (node.getEscape().isPresent()) { RowExpression escape = process(node.getEscape().get(), context); - return likeFunctionCall(value, call("LIKE_PATTERN", standardFunctionResolution.likePatternFunction(), LIKE_PATTERN, pattern, escape)); + return likeFunctionCall(value, call("LIKE_PATTERN", functionResolution.likePatternFunction(), LIKE_PATTERN, pattern, escape)); } return likeFunctionCall(value, call(CAST.name(), functionManager.lookupCast(CAST, VARCHAR.getTypeSignature(), LIKE_PATTERN.getTypeSignature()), LIKE_PATTERN, pattern)); @@ -755,11 +754,11 @@ protected RowExpression visitLikePredicate(LikePredicate node, Void context) private RowExpression likeFunctionCall(RowExpression value, RowExpression pattern) { if (value.getType() instanceof VarcharType) { - return call("LIKE", standardFunctionResolution.likeVarcharFunction(), BOOLEAN, value, pattern); + return call("LIKE", functionResolution.likeVarcharFunction(), BOOLEAN, value, pattern); } checkState(value.getType() instanceof CharType, "LIKE value type is neither VARCHAR or CHAR"); - return call("LIKE", standardFunctionResolution.likeCharFunction(value.getType()), BOOLEAN, value, pattern); + return call("LIKE", functionResolution.likeCharFunction(value.getType()), BOOLEAN, value, pattern); } @Override @@ -785,7 +784,7 @@ protected RowExpression visitArrayConstructor(ArrayConstructor node, Void contex List argumentTypes = arguments.stream() .map(RowExpression::getType) .collect(toImmutableList()); - return call("ARRAY", standardFunctionResolution.arrayConstructor(argumentTypes), getType(node), arguments); + return call("ARRAY", functionResolution.arrayConstructor(argumentTypes), getType(node), arguments); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java index 25ba9d550481b..c277c4af3dbc5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java @@ -13,12 +13,11 @@ */ package com.facebook.presto.sql.relational.optimizer; -import com.facebook.presto.Session; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; @@ -62,10 +61,10 @@ public class ExpressionOptimizer private final FunctionManager functionManager; private final ConnectorSession session; - public ExpressionOptimizer(FunctionManager functionManager, Session session) + public ExpressionOptimizer(FunctionManager functionManager, ConnectorSession session) { this.functionManager = functionManager; - this.session = session.toConnectorSession(); + this.session = session; } public RowExpression optimize(RowExpression expression) diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 53fd41b5775bc..8de824023820c 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -123,6 +123,7 @@ import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; import com.facebook.presto.sql.gen.OrderingCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; +import com.facebook.presto.sql.gen.RowExpressionPredicateCompiler; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.LocalExecutionPlanner; import com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan; @@ -139,6 +140,7 @@ import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.planPrinter.PlanPrinter; import com.facebook.presto.sql.planner.sanity.PlanSanityChecker; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateView; @@ -342,7 +344,9 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, typeRegistry, pageSorter, pageIndexerFactory, - transactionManager); + transactionManager, + new RowExpressionDomainTranslator(metadata), + new RowExpressionPredicateCompiler(metadata)); GlobalSystemConnectorFactory globalSystemConnectorFactory = new GlobalSystemConnectorFactory(ImmutableSet.of( new NodeSystemTable(nodeManager), diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index 916c94e950244..2b9a90565dac3 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -20,15 +20,22 @@ import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.InMemoryNodeManager; +import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.JoinCompiler; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.type.TypeRegistry; public class TestingConnectorContext @@ -36,14 +43,12 @@ public class TestingConnectorContext { private final NodeManager nodeManager = new ConnectorAwareNodeManager(new InMemoryNodeManager(), "testenv", new ConnectorId("test")); private final TypeManager typeManager = new TypeRegistry(); + private final FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + private final StandardFunctionResolution functionResolution = new FunctionResolution(functionManager); private final PageSorter pageSorter = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); private final PageIndexerFactory pageIndexerFactory = new GroupByHashPageIndexerFactory(new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())); - - public TestingConnectorContext() - { - // associate typeManager with a function manager - new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - } + private final Metadata metadata = MetadataManager.createTestMetadataManager(); + private final DomainTranslator domainTranslator = new RowExpressionDomainTranslator(metadata); @Override public NodeManager getNodeManager() @@ -57,6 +62,18 @@ public TypeManager getTypeManager() return typeManager; } + @Override + public FunctionMetadataManager getFunctionMetadataManager() + { + return functionManager; + } + + @Override + public StandardFunctionResolution getStandardFunctionResolution() + { + return functionResolution; + } + @Override public PageSorter getPageSorter() { @@ -68,4 +85,17 @@ public PageIndexerFactory getPageIndexerFactory() { return pageIndexerFactory; } + + @Override + public RowExpressionService getRowExpressionService() + { + return new RowExpressionService() + { + @Override + public DomainTranslator getDomainTranslator() + { + return domainTranslator; + } + }; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingNodeManager.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingNodeManager.java index ca605fd95cf9d..d76239c726078 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingNodeManager.java @@ -14,7 +14,7 @@ package com.facebook.presto.testing; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeManager; import com.google.common.collect.ImmutableSet; @@ -43,7 +43,7 @@ public TestingNodeManager() public TestingNodeManager(String environment) { - this(environment, new PrestoNode("local", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true), ImmutableSet.of()); + this(environment, new InternalNode("local", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true), ImmutableSet.of()); } public TestingNodeManager(Node localNode) diff --git a/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java b/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java index a3b0e7e1a2505..8427094a3c58b 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java +++ b/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java @@ -14,6 +14,7 @@ package com.facebook.presto.block; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.block.AbstractMapBlock.HashTables; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.BlockBuilderStatus; @@ -70,6 +71,9 @@ public abstract class AbstractTestBlock protected void assertBlock(Block block, Supplier newBlockBuilder, T[] expectedValues) { + assertBlockSize(block); + assertRetainedSize(block); + assertBlockPositions(block, newBlockBuilder, expectedValues); assertBlockPositions(copyBlockViaBlockSerde(block), newBlockBuilder, expectedValues); assertBlockPositions(copyBlockViaWritePositionTo(block, newBlockBuilder), newBlockBuilder, expectedValues); @@ -154,6 +158,9 @@ else if (type == short[].class) { else if (type == DictionaryId.class) { retainedSize += ClassLayout.parseClass(DictionaryId.class).instanceSize(); } + else if (type == HashTables.class) { + retainedSize += ((HashTables) field.get(block)).getRetainedSizeInBytes(); + } else if (type == MethodHandle.class) { // MethodHandles are only used in MapBlock/MapBlockBuilder, // and they are shared among blocks created by the same MapType. diff --git a/presto-main/src/test/java/com/facebook/presto/block/TestMapBlock.java b/presto-main/src/test/java/com/facebook/presto/block/TestMapBlock.java index 83edde3abc9cc..0b7659f784e39 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/TestMapBlock.java +++ b/presto-main/src/test/java/com/facebook/presto/block/TestMapBlock.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.ByteArrayBlock; +import com.facebook.presto.spi.block.MapBlock; import com.facebook.presto.spi.block.MapBlockBuilder; import com.facebook.presto.spi.block.SingleMapBlock; import com.facebook.presto.spi.type.MapType; @@ -27,6 +28,7 @@ import org.testng.annotations.Test; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -80,6 +82,46 @@ public void testCompactBlock() testIncompactBlock(mapType(TINYINT, TINYINT).createBlockFromKeyValue(Optional.of(mapIsNull), offsets, inCompactKeyBlock, inCompactValueBlock)); } + // TODO: remove this test when we have a more unified testWith() using assertBlock() + @Test + public void testLazyHashTableBuildOverBlockRegion() + { + Map[] values = createTestMap(9, 3, 4, 0, 8, 0, 6, 5); + Block block = createBlockWithValuesFromKeyValueBlock(values); + BlockBuilder blockBuilder = createBlockBuilderWithValues(values); + + // Create a MapBlock that is a region of another MapBlock. It doesn't have hashtables built at the time of creation. + int offset = block.getPositionCount() / 2; + Block blockRegion = block.getRegion(offset, block.getPositionCount() - offset); + + assertFalse(((MapBlock) blockRegion).isHashTablesPresent()); + assertFalse(((MapBlock) block).isHashTablesPresent()); + + // Lazily build the hashtables for the block region and use them to do position/value check. + Map[] expectedValues = Arrays.copyOfRange(values, values.length / 2, values.length); + assertBlock(blockRegion, () -> blockBuilder.newBlockBuilderLike(null), expectedValues); + + assertTrue(((MapBlock) blockRegion).isHashTablesPresent()); + assertTrue(((MapBlock) block).isHashTablesPresent()); + + Map[] valuesWithNull = alternatingNullValues(values); + Block blockWithNull = createBlockWithValuesFromKeyValueBlock(valuesWithNull); + + // Create a MapBlock that is a region of another MapBlock with null values. It doesn't have hashtables built at the time of creation. + offset = blockWithNull.getPositionCount() / 2; + Block blockRegionWithNull = blockWithNull.getRegion(offset, blockWithNull.getPositionCount() - offset); + + assertFalse(((MapBlock) blockRegionWithNull).isHashTablesPresent()); + assertFalse(((MapBlock) blockWithNull).isHashTablesPresent()); + + // Lazily build the hashtables for the block region and use them to do position/value check. + Map[] expectedValuesWithNull = Arrays.copyOfRange(valuesWithNull, valuesWithNull.length / 2, valuesWithNull.length); + assertBlock(blockRegionWithNull, () -> blockBuilder.newBlockBuilderLike(null), expectedValuesWithNull); + + assertTrue(((MapBlock) blockRegionWithNull).isHashTablesPresent()); + assertTrue(((MapBlock) blockWithNull).isHashTablesPresent()); + } + private Map[] createTestMap(int... entryCounts) { Map[] result = new Map[entryCounts.length]; diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java index 8587f911313dc..7ee8b02af4445 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java @@ -766,7 +766,7 @@ private AggregationNode aggregation(String id, PlanNode source) { AggregationNode.Aggregation aggregation = new AggregationNode.Aggregation( new FunctionCall(QualifiedName.of("count"), ImmutableList.of()), - metadata.getFunctionManager().lookupFunction(QualifiedName.of("count"), ImmutableList.of()), + metadata.getFunctionManager().lookupFunction("count", ImmutableList.of()), Optional.empty()); return new AggregationNode( diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestFilterStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestFilterStatsCalculator.java index bd1aaf2b8cd07..b69efc90334b1 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestFilterStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestFilterStatsCalculator.java @@ -158,6 +158,27 @@ public void testBooleanLiteralStats() .symbolStats("rightOpen", SymbolStatsAssertion::empty) .symbolStats("emptyRange", SymbolStatsAssertion::empty) .symbolStats("unknownRange", SymbolStatsAssertion::empty); + + // `null AND null` is interpreted as null + assertExpression("cast(null as boolean) AND cast(null as boolean)") + .outputRowsCount(0.0) + .symbolStats("x", SymbolStatsAssertion::empty) + .symbolStats("y", SymbolStatsAssertion::empty) + .symbolStats("z", SymbolStatsAssertion::empty) + .symbolStats("leftOpen", SymbolStatsAssertion::empty) + .symbolStats("rightOpen", SymbolStatsAssertion::empty) + .symbolStats("emptyRange", SymbolStatsAssertion::empty) + .symbolStats("unknownRange", SymbolStatsAssertion::empty); + + // more complicated expressions with null + assertExpression("cast(null as boolean) OR sin(x) > x").outputRowsCount(NaN); + + // RowExpressionInterpreter does more aggressive constant folding than ExpressionInterpreter. + // For ExpressionInterpreter, cast will return unknown; but RowExpressionInterpreter can actually evaluate cast. + Expression expression = expression("cast(null as boolean) AND sin(x) > x"); + RowExpression rowExpression = translator.translateAndOptimize(expression, standardTypes); + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(standardInputStatistics, rowExpression, session, standardTypes); + PlanNodeStatsAssertion.assertThat(rowExpressionStatsEstimate).outputRowsCount(0.0); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java b/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java index 306afbc990661..e0c8799f8f230 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java @@ -14,7 +14,7 @@ package com.facebook.presto.cost; import com.facebook.presto.sql.planner.Symbol; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -35,7 +35,7 @@ public class TestValuesNodeStats @Test public void testStatsForValuesNode() { - StandardFunctionResolution resolution = new StandardFunctionResolution(tester().getMetadata().getFunctionManager()); + FunctionResolution resolution = new FunctionResolution(tester().getMetadata().getFunctionManager()); tester().assertStatsFor(pb -> pb .values(ImmutableList.of(pb.symbol("a", BIGINT), pb.symbol("b", DOUBLE)), ImmutableList.of( @@ -86,7 +86,7 @@ public void testStatsForValuesNode() @Test public void testStatsForValuesNodeWithJustNulls() { - StandardFunctionResolution resolution = new StandardFunctionResolution(tester().getMetadata().getFunctionManager()); + FunctionResolution resolution = new FunctionResolution(tester().getMetadata().getFunctionManager()); PlanNodeStatsEstimate nullAStats = PlanNodeStatsEstimate.builder() .setOutputRowCount(1) .addSymbolStatistics(new Symbol("a"), SymbolStatsEstimate.zero()) diff --git a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java index 605e1e588bcdc..810d1352005a7 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java @@ -23,11 +23,10 @@ import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; import com.facebook.presto.execution.scheduler.NodeSelector; import com.facebook.presto.metadata.InMemoryNodeManager; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.testing.TestingTransactionHandle; import com.facebook.presto.util.FinalizerService; @@ -100,8 +99,8 @@ public Object benchmark(BenchmarkData data) Iterator splits = data.getSplits().iterator(); Set batch = new HashSet<>(); while (splits.hasNext() || !batch.isEmpty()) { - Multimap assignments = data.getNodeSelector().computeAssignments(batch, remoteTasks).getAssignments(); - for (Node node : assignments.keySet()) { + Multimap assignments = data.getNodeSelector().computeAssignments(batch, remoteTasks).getAssignments(); + for (InternalNode node : assignments.keySet()) { MockRemoteTaskFactory.MockRemoteTask remoteTask = data.getTaskMap().get(node); remoteTask.addSplits(ImmutableMultimap.builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) @@ -134,7 +133,7 @@ public static class BenchmarkData private FinalizerService finalizerService = new FinalizerService(); private NodeSelector nodeSelector; - private Map taskMap = new HashMap<>(); + private Map taskMap = new HashMap<>(); private List splits = new ArrayList<>(); @Setup @@ -145,16 +144,16 @@ public void setup() finalizerService.start(); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); - ImmutableList.Builder nodeBuilder = ImmutableList.builder(); + ImmutableList.Builder nodeBuilder = ImmutableList.builder(); for (int i = 0; i < NODES; i++) { - nodeBuilder.add(new PrestoNode("node" + i, URI.create("http://" + addressForHost(i).getHostText()), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("node" + i, URI.create("http://" + addressForHost(i).getHostText()), NodeVersion.UNKNOWN, false)); } - List nodes = nodeBuilder.build(); + List nodes = nodeBuilder.build(); MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory( newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s")), newScheduledThreadPool(2, daemonThreadsNamed("remoteTaskScheduledExecutor-%s"))); for (int i = 0; i < nodes.size(); i++) { - Node node = nodes.get(i); + InternalNode node = nodes.get(i); ImmutableList.Builder initialSplits = ImmutableList.builder(); for (int j = 0; j < MAX_SPLITS_PER_NODE + MAX_PENDING_SPLITS_PER_TASK_PER_NODE; j++) { initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(i))); @@ -209,7 +208,7 @@ private NetworkTopology getNetworkTopology() return topology; } - public Map getTaskMap() + public Map getTaskMap() { return taskMap; } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java index 747473c49ea66..e2bbd622ae789 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java @@ -23,12 +23,12 @@ import com.facebook.presto.memory.MemoryPool; import com.facebook.presto.memory.QueryContext; import com.facebook.presto.memory.context.SimpleLocalMemoryContext; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.metadata.TableHandle; import com.facebook.presto.operator.StageExecutionDescriptor; import com.facebook.presto.operator.TaskContext; import com.facebook.presto.operator.TaskStats; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.memory.MemoryPoolId; import com.facebook.presto.spiller.SpillSpaceTracker; import com.facebook.presto.sql.planner.Partitioning; @@ -102,7 +102,7 @@ public MockRemoteTaskFactory(Executor executor, ScheduledExecutorService schedul this.scheduledExecutor = scheduledExecutor; } - public MockRemoteTask createTableScanTask(TaskId taskId, Node newNode, List splits, PartitionedSplitCountTracker partitionedSplitCountTracker) + public MockRemoteTask createTableScanTask(TaskId taskId, InternalNode newNode, List splits, PartitionedSplitCountTracker partitionedSplitCountTracker) { Symbol symbol = new Symbol("column"); PlanNodeId sourceId = new PlanNodeId("sourceId"); @@ -132,7 +132,7 @@ public MockRemoteTask createTableScanTask(TaskId taskId, Node newNode, List initialSplits, OptionalInt totalPartitions, @@ -143,12 +143,6 @@ public MockRemoteTask createRemoteTask( return new MockRemoteTask(taskId, fragment, node.getNodeIdentifier(), executor, scheduledExecutor, initialSplits, totalPartitions, partitionedSplitCountTracker); } - @Override - public ListenableFuture removeRemoteSource(TaskId taskId, TaskId remoteSourceTaskId) - { - throw new UnsupportedOperationException(); - } - public static final class MockRemoteTask implements RemoteTask { @@ -374,6 +368,12 @@ public void setOutputBuffers(OutputBuffers outputBuffers) outputBuffer.setOutputBuffers(outputBuffers); } + @Override + public ListenableFuture removeRemoteSource(TaskId remoteSourceTaskId) + { + throw new UnsupportedOperationException(); + } + @Override public void addStateChangeListener(StateChangeListener stateChangeListener) { diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java index a4e8c153e0fb4..65a0aebc1156e 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java @@ -23,11 +23,10 @@ import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; import com.facebook.presto.execution.scheduler.NodeSelector; import com.facebook.presto.metadata.InMemoryNodeManager; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.testing.TestingTransactionHandle; import com.facebook.presto.util.FinalizerService; @@ -73,7 +72,7 @@ public class TestNodeScheduler private NodeTaskMap nodeTaskMap; private InMemoryNodeManager nodeManager; private NodeSelector nodeSelector; - private Map taskMap; + private Map taskMap; private ExecutorService remoteTaskExecutor; private ScheduledExecutorService remoteTaskScheduledExecutor; @@ -84,11 +83,11 @@ public void setUp() nodeTaskMap = new NodeTaskMap(finalizerService); nodeManager = new InMemoryNodeManager(); - ImmutableList.Builder nodeBuilder = ImmutableList.builder(); - nodeBuilder.add(new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)); - nodeBuilder.add(new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false)); - nodeBuilder.add(new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); - ImmutableList nodes = nodeBuilder.build(); + ImmutableList.Builder nodeBuilder = ImmutableList.builder(); + nodeBuilder.add(new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); + ImmutableList nodes = nodeBuilder.build(); nodeManager.addNode(CONNECTOR_ID, nodes); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() .setMaxSplitsPerNode(20) @@ -119,7 +118,7 @@ public void testScheduleLocal() Split split = new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitLocal()); Set splits = ImmutableSet.of(split); - Map.Entry assignment = Iterables.getOnlyElement(nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments().entries()); + Map.Entry assignment = Iterables.getOnlyElement(nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments().entries()); assertEquals(assignment.getKey().getHostAndPort(), split.getAddresses().get(0)); assertEquals(assignment.getValue(), split); } @@ -132,15 +131,15 @@ public void testTopologyAwareScheduling() NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); InMemoryNodeManager nodeManager = new InMemoryNodeManager(); - ImmutableList.Builder nodeBuilder = ImmutableList.builder(); - nodeBuilder.add(new PrestoNode("node1", URI.create("http://host1.rack1:11"), NodeVersion.UNKNOWN, false)); - nodeBuilder.add(new PrestoNode("node2", URI.create("http://host2.rack1:12"), NodeVersion.UNKNOWN, false)); - nodeBuilder.add(new PrestoNode("node3", URI.create("http://host3.rack2:13"), NodeVersion.UNKNOWN, false)); - ImmutableList nodes = nodeBuilder.build(); + ImmutableList.Builder nodeBuilder = ImmutableList.builder(); + nodeBuilder.add(new InternalNode("node1", URI.create("http://host1.rack1:11"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("node2", URI.create("http://host2.rack1:12"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("node3", URI.create("http://host3.rack2:13"), NodeVersion.UNKNOWN, false)); + ImmutableList nodes = nodeBuilder.build(); nodeManager.addNode(CONNECTOR_ID, nodes); // contents of taskMap indicate the node-task map for the current stage - Map taskMap = new HashMap<>(); + Map taskMap = new HashMap<>(); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() .setMaxSplitsPerNode(25) .setIncludeCoordinator(false) @@ -171,10 +170,10 @@ public NetworkLocation get(HostAddress host) nonRackLocalBuilder.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromParts("data.other_rack", 1)))); } Set nonRackLocalSplits = nonRackLocalBuilder.build(); - Multimap assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())).getAssignments(); MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); int task = 0; - for (Node node : assignments.keySet()) { + for (InternalNode node : assignments.keySet()) { TaskId taskId = new TaskId("test", 1, task); task++; MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, ImmutableList.copyOf(assignments.get(node)), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); @@ -185,7 +184,7 @@ public NetworkLocation get(HostAddress host) // Continue assigning to fill up part of the queue nonRackLocalSplits = Sets.difference(nonRackLocalSplits, new HashSet<>(assignments.values())); assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())).getAssignments(); - for (Node node : assignments.keySet()) { + for (InternalNode node : assignments.keySet()) { RemoteTask remoteTask = taskMap.get(node); remoteTask.addSplits(ImmutableMultimap.builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) @@ -206,7 +205,7 @@ public NetworkLocation get(HostAddress host) rackLocalSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(dataHost2))); } assignments = nodeSelector.computeAssignments(rackLocalSplits.build(), ImmutableList.copyOf(taskMap.values())).getAssignments(); - for (Node node : assignments.keySet()) { + for (InternalNode node : assignments.keySet()) { RemoteTask remoteTask = taskMap.get(node); remoteTask.addSplits(ImmutableMultimap.builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) @@ -227,7 +226,7 @@ public NetworkLocation get(HostAddress host) MILLISECONDS.sleep(10); } assignments = nodeSelector.computeAssignments(unassigned, ImmutableList.copyOf(taskMap.values())).getAssignments(); - for (Node node : assignments.keySet()) { + for (InternalNode node : assignments.keySet()) { RemoteTask remoteTask = taskMap.get(node); remoteTask.addSplits(ImmutableMultimap.builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) @@ -268,7 +267,7 @@ public void testScheduleRemote() { Set splits = new HashSet<>(); splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())); - Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); assertEquals(assignments.size(), 1); } @@ -280,9 +279,9 @@ public void testBasicAssignment() for (int i = 0; i < 3; i++) { splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())); } - Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); assertEquals(assignments.entries().size(), 3); - for (Node node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { + for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { assertTrue(assignments.keySet().contains(node)); } } @@ -292,7 +291,7 @@ public void testMaxSplitsPerNode() { TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); - Node newNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + InternalNode newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, newNode); ImmutableList.Builder initialSplits = ImmutableList.builder(); @@ -314,7 +313,7 @@ public void testMaxSplitsPerNode() for (int i = 0; i < 5; i++) { splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); } - Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); // no split should be assigned to the newNode, as it already has maxNodeSplits assigned to it assertFalse(assignments.keySet().contains(newNode)); @@ -330,7 +329,7 @@ public void testMaxSplitsPerNodePerTask() { TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); - Node newNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + InternalNode newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, newNode); ImmutableList.Builder initialSplits = ImmutableList.builder(); @@ -340,7 +339,7 @@ public void testMaxSplitsPerNodePerTask() List tasks = new ArrayList<>(); MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); - for (Node node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { + for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { // Max out number of splits on node TaskId taskId = new TaskId("test", 1, 1); RemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); @@ -359,7 +358,7 @@ public void testMaxSplitsPerNodePerTask() for (int i = 0; i < 5; i++) { splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); } - Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); // no split should be assigned to the newNode, as it already has // maxSplitsPerNode + maxSplitsPerNodePerTask assigned to it @@ -377,7 +376,7 @@ public void testTaskCompletion() throws Exception { MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); - Node chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); + InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); TaskId taskId = new TaskId("test", 1, 1); RemoteTask remoteTask = remoteTaskFactory.createTableScanTask( taskId, @@ -398,7 +397,7 @@ public void testTaskCompletion() public void testSplitCount() { MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); - Node chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); + InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); TaskId taskId1 = new TaskId("test", 1, 1); RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java index 4e7e1a2902c48..48ee3b2fe3879 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java @@ -18,9 +18,8 @@ import com.facebook.presto.execution.TestSqlTaskManager.MockLocationFactory; import com.facebook.presto.execution.scheduler.SplitSchedulerStats; import com.facebook.presto.failureDetector.NoOpFailureDetector; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.operator.StageExecutionDescriptor; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.Partitioning; @@ -126,7 +125,7 @@ private void testFinalStageInfoInternal() if (Thread.interrupted()) { return; } - Node node = new PrestoNode( + InternalNode node = new InternalNode( "source" + i, URI.create("http://10.0.0." + (i / 10_000) + ":" + (i % 10_000)), NodeVersion.UNKNOWN, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java index 7a6387a454ef4..0c4745f3216c7 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java @@ -21,9 +21,9 @@ import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.memory.NodeMemoryConfig; import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.operator.ExchangeClient; import com.facebook.presto.operator.ExchangeClientSupplier; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spiller.LocalSpillManager; import com.facebook.presto.spiller.NodeSpillConfig; @@ -310,13 +310,13 @@ public URI createLocalTaskLocation(TaskId taskId) } @Override - public URI createTaskLocation(Node node, TaskId taskId) + public URI createTaskLocation(InternalNode node, TaskId taskId) { return URI.create("http://fake.invalid/task/" + node.getNodeIdentifier() + "/" + taskId); } @Override - public URI createMemoryInfoLocation(Node node) + public URI createMemoryInfoLocation(InternalNode node) { return URI.create("http://fake.invalid/" + node.getNodeIdentifier() + "/memory"); } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java index 03a38e08067e5..8fcc8a69253ec 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java @@ -18,8 +18,7 @@ import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.TaskId; -import com.facebook.presto.metadata.PrestoNode; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -91,10 +90,10 @@ public void testMultipleNodes() assertEquals(result.getNewTasks().stream().map(RemoteTask::getNodeId).collect(toImmutableSet()).size(), 5); } - private static List generateRandomNodes(int count) + private static List generateRandomNodes(int count) { return IntStream.range(0, count) - .mapToObj(i -> new PrestoNode("other " + i, URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)) + .mapToObj(i -> new InternalNode("other " + i, URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)) .collect(toImmutableList()); } } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java index c0f502a0a792b..160be40f3d2ac 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java @@ -27,14 +27,13 @@ import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.metadata.InMemoryNodeManager; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; -import com.facebook.presto.metadata.PrestoNode; import com.facebook.presto.metadata.TableHandle; import com.facebook.presto.operator.StageExecutionDescriptor; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; import com.facebook.presto.spi.FixedSplitSource; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; @@ -108,9 +107,9 @@ public class TestSourcePartitionedScheduler public TestSourcePartitionedScheduler() { nodeManager.addNode(CONNECTOR_ID, - new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), - new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), - new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); + new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), + new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), + new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); } @BeforeClass @@ -335,9 +334,9 @@ public void testBalancedSplitAssignment() // use private node manager so we can add a node later InMemoryNodeManager nodeManager = new InMemoryNodeManager(); nodeManager.addNode(CONNECTOR_ID, - new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), - new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), - new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); + new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), + new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), + new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); // Schedule 15 splits - there are 3 nodes, each node should get 5 splits @@ -355,7 +354,7 @@ public void testBalancedSplitAssignment() } // Add new node - Node additionalNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + InternalNode additionalNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, additionalNode); // Schedule 5 splits in another query. Since the new node does not have any splits, all 5 splits are assigned to the new node diff --git a/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java b/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java index b29cceaf63c6e..e3f3650e89669 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java @@ -15,8 +15,7 @@ package com.facebook.presto.memory; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.memory.MemoryPoolId; import com.facebook.presto.spi.memory.MemoryPoolInfo; @@ -39,14 +38,14 @@ private LowMemoryKillerTestingUtils() {} static List toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map> queries) { - Map nodeReservations = new HashMap<>(); + Map nodeReservations = new HashMap<>(); for (Map.Entry> entry : queries.entrySet()) { QueryId queryId = new QueryId(entry.getKey()); Map reservationByNode = entry.getValue(); for (Map.Entry nodeEntry : reservationByNode.entrySet()) { - PrestoNode node = new PrestoNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false); + InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false); long bytes = nodeEntry.getValue(); if (bytes == 0) { continue; @@ -61,7 +60,7 @@ static List toNodeMemoryInfoList(long maxReservedPoolBytes, long max } ImmutableList.Builder result = ImmutableList.builder(); - for (Map.Entry entry : nodeReservations.entrySet()) { + for (Map.Entry entry : nodeReservations.entrySet()) { NodeReservation nodeReservation = entry.getValue(); ImmutableMap.Builder pools = ImmutableMap.builder(); if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) { diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java index 6c023425ef9a9..785170957bad2 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java @@ -16,7 +16,6 @@ import com.facebook.presto.client.NodeVersion; import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.server.InternalCommunicationConfig; -import com.facebook.presto.spi.Node; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -56,10 +55,10 @@ public class TestDiscoveryNodeManager private final NodeInfo nodeInfo = new NodeInfo("test"); private final InternalCommunicationConfig internalCommunicationConfig = new InternalCommunicationConfig(); private NodeVersion expectedVersion; - private Set activeNodes; - private Set inactiveNodes; - private PrestoNode coordinator; - private PrestoNode currentNode; + private Set activeNodes; + private Set inactiveNodes; + private InternalNode coordinator; + private InternalNode currentNode; private final PrestoNodeServiceSelector selector = new PrestoNodeServiceSelector(); private HttpClient testHttpClient; @@ -69,17 +68,17 @@ public void setup() testHttpClient = new TestingHttpClient(input -> new TestingResponse(OK, ArrayListMultimap.create(), ACTIVE.name().getBytes())); expectedVersion = new NodeVersion("1"); - coordinator = new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.2.8"), expectedVersion, true); - currentNode = new PrestoNode(nodeInfo.getNodeId(), URI.create("http://192.0.1.1"), expectedVersion, false); + coordinator = new InternalNode(UUID.randomUUID().toString(), URI.create("https://192.0.2.8"), expectedVersion, true); + currentNode = new InternalNode(nodeInfo.getNodeId(), URI.create("http://192.0.1.1"), expectedVersion, false); activeNodes = ImmutableSet.of( currentNode, - new PrestoNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.1:8080"), expectedVersion, false), - new PrestoNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.3"), expectedVersion, false), + new InternalNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.1:8080"), expectedVersion, false), + new InternalNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.3"), expectedVersion, false), coordinator); inactiveNodes = ImmutableSet.of( - new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.3.9"), NodeVersion.UNKNOWN, false), - new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.4.9"), new NodeVersion("2"), false)); + new InternalNode(UUID.randomUUID().toString(), URI.create("https://192.0.3.9"), NodeVersion.UNKNOWN, false), + new InternalNode(UUID.randomUUID().toString(), URI.create("https://192.0.4.9"), new NodeVersion("2"), false)); selector.announceNodes(activeNodes, inactiveNodes); } @@ -91,22 +90,22 @@ public void testGetAllNodes() try { AllNodes allNodes = manager.getAllNodes(); - Set activeNodes = allNodes.getActiveNodes(); + Set activeNodes = allNodes.getActiveNodes(); assertEqualsIgnoreOrder(activeNodes, this.activeNodes); - for (Node actual : activeNodes) { - for (Node expected : this.activeNodes) { + for (InternalNode actual : activeNodes) { + for (InternalNode expected : this.activeNodes) { assertNotSame(actual, expected); } } assertEqualsIgnoreOrder(activeNodes, manager.getNodes(ACTIVE)); - Set inactiveNodes = allNodes.getInactiveNodes(); + Set inactiveNodes = allNodes.getInactiveNodes(); assertEqualsIgnoreOrder(inactiveNodes, this.inactiveNodes); - for (Node actual : inactiveNodes) { - for (Node expected : this.inactiveNodes) { + for (InternalNode actual : inactiveNodes) { + for (InternalNode expected : this.inactiveNodes) { assertNotSame(actual, expected); } } @@ -188,14 +187,14 @@ public static class PrestoNodeServiceSelector @GuardedBy("this") private List descriptors = ImmutableList.of(); - private synchronized void announceNodes(Set activeNodes, Set inactiveNodes) + private synchronized void announceNodes(Set activeNodes, Set inactiveNodes) { ImmutableList.Builder descriptors = ImmutableList.builder(); - for (Node node : Iterables.concat(activeNodes, inactiveNodes)) { + for (InternalNode node : Iterables.concat(activeNodes, inactiveNodes)) { descriptors.add(serviceDescriptor("presto") .setNodeId(node.getNodeIdentifier()) - .addProperty("http", node.getHttpUri().toString()) - .addProperty("node_version", ((PrestoNode) node).getNodeVersion().toString()) + .addProperty("http", node.getInternalUri().toString()) + .addProperty("node_version", node.getNodeVersion().toString()) .addProperty("coordinator", String.valueOf(node.isCoordinator())) .build()); } diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java index 468610164df0f..b016e5c134fe1 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java @@ -27,7 +27,7 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; @@ -71,7 +71,7 @@ public void testIdentityCast() TypeRegistry typeManager = new TypeRegistry(); StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); FunctionHandle exactOperator = staticFunctionNamespace.lookupCast(CastType.CAST, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()); - assertEquals(exactOperator, new FunctionHandle(new Signature(mangleOperatorName(CAST.name()), SCALAR, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()))); + assertEquals(exactOperator, new StaticFunctionHandle(new Signature(mangleOperatorName(CAST.name()), SCALAR, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()))); } @Test @@ -91,7 +91,7 @@ public void testExactMatchBeforeCoercion() if (function.getSignature().getArgumentTypes().stream().anyMatch(TypeSignature::isCalculated)) { continue; } - FunctionHandle exactOperator = staticFunctionNamespace.resolveOperator(operatorType, fromTypeSignatures(function.getSignature().getArgumentTypes())); + StaticFunctionHandle exactOperator = (StaticFunctionHandle) staticFunctionNamespace.resolveOperator(operatorType, fromTypeSignatures(function.getSignature().getArgumentTypes())); assertEquals(exactOperator.getSignature(), function.getSignature()); foundOperator = true; } @@ -108,7 +108,7 @@ public void testMagicLiteralFunction() TypeRegistry typeManager = new TypeRegistry(); StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); - FunctionHandle functionHandle = staticFunctionNamespace.resolveFunction(QualifiedName.of(signature.getName()), fromTypeSignatures(signature.getArgumentTypes())); + StaticFunctionHandle functionHandle = (StaticFunctionHandle) staticFunctionNamespace.resolveFunction(QualifiedName.of(signature.getName()), fromTypeSignatures(signature.getArgumentTypes())); assertEquals(staticFunctionNamespace.getFunctionMetadata(functionHandle).getArgumentTypes(), ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT))); assertEquals(signature.getReturnType().getBase(), StandardTypes.TIMESTAMP_WITH_TIME_ZONE); } @@ -162,7 +162,7 @@ public void testOperatorTypes() { TypeRegistry typeManager = new TypeRegistry(); FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - StandardFunctionResolution functionResolution = new StandardFunctionResolution(functionManager); + FunctionResolution functionResolution = new FunctionResolution(functionManager); assertTrue(functionManager.getFunctionMetadata(functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT)).getOperatorType().map(OperatorType::isArithmeticOperator).orElse(false)); assertFalse(functionManager.getFunctionMetadata(functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT)).getOperatorType().map(OperatorType::isComparisonOperator).orElse(true)); @@ -372,7 +372,7 @@ public ResolveFunctionAssertion forParameters(String... parameters) public ResolveFunctionAssertion returns(SignatureBuilder functionSignature) { - FunctionHandle expectedFunction = new FunctionHandle(functionSignature.name(TEST_FUNCTION_NAME).build()); + FunctionHandle expectedFunction = new StaticFunctionHandle(functionSignature.name(TEST_FUNCTION_NAME).build()); FunctionHandle actualFunction = resolveFunctionHandle(); assertEquals(expectedFunction, actualFunction); return this; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java index 74bee8a69d16c..5de06019876f0 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java @@ -26,7 +26,6 @@ import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.TestingTaskContext; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; @@ -84,9 +83,9 @@ public class BenchmarkHashAndStreamingAggregationOperators private static final FunctionManager functionManager = metadata.getFunctionManager(); private static final InternalAggregationFunction LONG_SUM = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT))); + functionManager.lookupFunction("sum", fromTypes(BIGINT))); private static final InternalAggregationFunction COUNT = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of())); + functionManager.lookupFunction("count", ImmutableList.of())); @State(Thread) public static class Context diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java index da375107f0f4d..13e9e704560b5 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.MaterializedResult; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterMethod; @@ -164,6 +163,6 @@ private void testMemoryTracking(boolean useSystemMemory) private static InternalAggregationFunction getAggregation(String name, Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(name, fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java index 2fb757448ec21..550d55dba16cb 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java @@ -365,8 +365,15 @@ public void testRemoveRemoteSource() // remove remote source while pages are still available exchangeClient.removeRemoteSource(taskId1); - // client should not receive any further pages from removed remote source - assertNull(exchangeClient.pollPage()); + SerializedPage nextPage = exchangeClient.pollPage(); + if (nextPage != null) { + // After the first page get polled, it is possible for the second page get buffered + // before remote source get removed. + assertPageEquals(nextPage, createPage(2)); + // Buffer capacity is set to 1 byte, so the third page cannot be buffered. + // Client should not receive any further pages from removed remote source. + assertNull(exchangeClient.pollPage()); + } assertEquals(exchangeClient.getStatus().getBufferedPages(), 0); // add pages to another source diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java index fd7ace6234ae2..30717a14b62ab 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java @@ -32,7 +32,6 @@ import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.TestingTaskContext; import com.google.common.collect.ImmutableList; @@ -100,7 +99,7 @@ public class TestHashAggregationOperator private static final InternalAggregationFunction LONG_AVERAGE = getAggregation("avg", BIGINT); private static final InternalAggregationFunction LONG_SUM = getAggregation("sum", BIGINT); private static final InternalAggregationFunction COUNT = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of())); + functionManager.lookupFunction("count", ImmutableList.of())); private static final int MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024; @@ -739,7 +738,7 @@ private int getHashCapacity(Operator operator) private static InternalAggregationFunction getAggregation(String name, Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(name, fromTypes(arguments))); } private static class DummySpillerFactory diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestRealAverageAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/TestRealAverageAggregation.java index 3393d8fe9683a..0ba0ededf17ae 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestRealAverageAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestRealAverageAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.StandardTypes; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -44,7 +43,7 @@ public void setUp() { FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); avgFunction = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("avg"), fromTypes(REAL))); + functionManager.lookupFunction("avg", fromTypes(REAL))); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java index e1001a1a0eb23..a2f5f4ae0d704 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java @@ -37,7 +37,6 @@ import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.TestingSplit; import com.facebook.presto.testing.TestingTransactionHandle; @@ -268,7 +267,7 @@ public void testPageYield() ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder projections = ImmutableList.builder(); for (int i = 0; i < totalColumns; i++) { - projections.add(call("generic_long_page_col", functionManager.lookupFunction(QualifiedName.of("generic_long_page_col" + i), fromTypes(BIGINT)), BIGINT, field(0, BIGINT))); + projections.add(call("generic_long_page_col", functionManager.lookupFunction("generic_long_page_col" + i, fromTypes(BIGINT)), BIGINT, field(0, BIGINT))); } Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections.build(), "key"); Supplier pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections.build(), MAX_BATCH_SIZE); @@ -333,7 +332,7 @@ public void testRecordCursorYield() List projections = ImmutableList.of(call( "generic_long_record_cursor", - functionManager.lookupFunction(QualifiedName.of("generic_long_record_cursor"), fromTypes(BIGINT)), + functionManager.lookupFunction("generic_long_record_cursor", fromTypes(BIGINT)), BIGINT, field(0, BIGINT))); Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key"); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java index 93dde1f24d068..02fb7dff61064 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java @@ -23,7 +23,6 @@ import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.MaterializedResult; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterMethod; @@ -54,9 +53,9 @@ public class TestStreamingAggregationOperator private static final FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); private static final InternalAggregationFunction LONG_SUM = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT))); + functionManager.lookupFunction("sum", fromTypes(BIGINT))); private static final InternalAggregationFunction COUNT = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of())); + functionManager.lookupFunction("count", ImmutableList.of())); private ExecutorService executor; private ScheduledExecutorService scheduledExecutor; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java index b4b7fdbcf1c9b..ff41a2ea4b6c6 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java @@ -27,7 +27,6 @@ import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.slice.Slice; @@ -66,7 +65,7 @@ public class TestTableFinishOperator { private static final FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); private static final InternalAggregationFunction LONG_MAX = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("max"), fromTypes(BIGINT))); + functionManager.lookupFunction("max", fromTypes(BIGINT))); private ScheduledExecutorService scheduledExecutor; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java index 4db39e78a7ca6..8f3cc2474fca3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java @@ -37,7 +37,6 @@ import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import org.testng.annotations.AfterClass; @@ -202,7 +201,7 @@ public void testStatisticsAggregation() .addDriverContext(); FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); InternalAggregationFunction longMaxFunction = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("max"), fromTypes(BIGINT))); + functionManager.lookupFunction("max", fromTypes(BIGINT))); TableWriterOperator operator = (TableWriterOperator) createTableWriterOperator( pageSinkManager, new AggregationOperatorFactory( diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkArrayAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkArrayAggregation.java index 8df07726c88ff..c5364d0dbb411 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkArrayAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkArrayAggregation.java @@ -19,7 +19,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -105,7 +104,7 @@ public void setup() } InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of(name), fromTypes(elementType))); + functionManager.lookupFunction(name, fromTypes(elementType))); accumulator = function.bind(ImmutableList.of(0), Optional.empty()).createAccumulator(); block = createChannel(ARRAY_SIZE, elementType); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkGroupedTypedHistogram.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkGroupedTypedHistogram.java index 2a46f554fa1bd..d66872bda282b 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkGroupedTypedHistogram.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkGroupedTypedHistogram.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.QualifiedName; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Fork; import org.openjdk.jmh.annotations.Measurement; @@ -158,7 +157,7 @@ private static InternalAggregationFunction getInternalAggregationFunctionVarChar FunctionManager functionManager = getMetadata(groupMode).getFunctionManager(); return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(VARCHAR))); + functionManager.lookupFunction(NAME, fromTypes(VARCHAR))); } private static MetadataManager getMetadata(HistogramGroupImplementation groupMode) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctBoolean.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctBoolean.java index 185b23ab72a3b..1def0b9d4465e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctBoolean.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctBoolean.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.primitives.Booleans; @@ -35,7 +34,7 @@ public class TestApproximateCountDistinctBoolean public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(BOOLEAN, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(BOOLEAN, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctDouble.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctDouble.java index 9d5d6af4e7084..fc4e59883382e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctDouble.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctDouble.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -28,7 +27,7 @@ public class TestApproximateCountDistinctDouble public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(DOUBLE, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(DOUBLE, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctInteger.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctInteger.java index 52d522f7fc677..7f7de8c15c7a7 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctInteger.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctInteger.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -29,7 +28,7 @@ public class TestApproximateCountDistinctInteger public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(INTEGER, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(INTEGER, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java index 09cca15a61684..d75f09ea0c1e1 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import io.airlift.slice.Slices; import java.util.concurrent.ThreadLocalRandom; @@ -30,7 +29,7 @@ public class TestApproximateCountDistinctIpAddress public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(IPADDRESS, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(IPADDRESS, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLong.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLong.java index d2052505aa0b8..27bd78302bd90 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLong.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLong.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -29,7 +28,7 @@ public class TestApproximateCountDistinctLong public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(BIGINT, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(BIGINT, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLongDecimal.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLongDecimal.java index a6767bd765a07..7afa1ab533440 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLongDecimal.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLongDecimal.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import io.airlift.slice.Slices; import java.util.concurrent.ThreadLocalRandom; @@ -33,7 +32,7 @@ public class TestApproximateCountDistinctLongDecimal public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(LONG_DECIMAL, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(LONG_DECIMAL, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctSmallint.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctSmallint.java index ae2518c6d38fb..7940c2530f657 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctSmallint.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctSmallint.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -29,7 +28,7 @@ public class TestApproximateCountDistinctSmallint public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(SMALLINT, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(SMALLINT, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctTinyint.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctTinyint.java index e07e52692d93f..4369d3a6fe0d9 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctTinyint.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctTinyint.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -29,7 +28,7 @@ public class TestApproximateCountDistinctTinyint public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(TINYINT, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(TINYINT, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctVarBinary.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctVarBinary.java index b71bd13c7b17a..4d0def5faa304 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctVarBinary.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctVarBinary.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import io.airlift.slice.Slices; import java.util.concurrent.ThreadLocalRandom; @@ -30,7 +29,7 @@ public class TestApproximateCountDistinctVarBinary public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(VARCHAR, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(VARCHAR, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximatePercentileAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximatePercentileAggregation.java index 38ab4bba02d0d..81853f970de43 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximatePercentileAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximatePercentileAggregation.java @@ -19,7 +19,6 @@ import com.facebook.presto.spi.block.RunLengthEncodedBlock; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -444,7 +443,7 @@ public void testDoublePartialStep() private static InternalAggregationFunction getAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("approx_percentile"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("approx_percentile", fromTypes(arguments))); } private static RunLengthEncodedBlock createRLEBlock(double percentile, int positionCount) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java index 48976549de8d0..4e22c7a4bca93 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java @@ -17,7 +17,6 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -167,6 +166,6 @@ public void testValidInt() private static InternalAggregationFunction getAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("arbitrary"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("arbitrary", fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java index cd09c43ca8f91..5c6979b507b52 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java @@ -24,7 +24,6 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.SqlDate; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; import org.testng.internal.collections.Ints; @@ -224,6 +223,6 @@ private GroupedAccumulator createGroupedAccumulator(InternalAggregationFunction private InternalAggregationFunction getAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("array_agg"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("array_agg", fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestChecksumAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestChecksumAggregation.java index eb082ddc16f24..453b58b906679 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestChecksumAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestChecksumAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import org.testng.annotations.Test; import static com.facebook.presto.block.BlockAssertions.createArrayBigintBlock; @@ -127,6 +126,6 @@ private static SqlVarbinary expectedChecksum(Type type, Block block) private InternalAggregationFunction getAggregation(Type argument) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("checksum"), fromTypes(argument))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("checksum", fromTypes(argument))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestDoubleHistogramAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestDoubleHistogramAggregation.java index 4285c477201b3..dbcbab76a7450 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestDoubleHistogramAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestDoubleHistogramAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.MapType; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import org.testng.annotations.Test; @@ -46,7 +45,7 @@ public TestDoubleHistogramAggregation() { FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("numeric_histogram"), fromTypes(BIGINT, DOUBLE, DOUBLE))); + functionManager.lookupFunction("numeric_histogram", fromTypes(BIGINT, DOUBLE, DOUBLE))); factory = function.bind(ImmutableList.of(0, 1, 2), Optional.empty()); input = makeInput(10); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestHistogram.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestHistogram.java index 75864541d7c66..b9732477781ad 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestHistogram.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestHistogram.java @@ -30,7 +30,6 @@ import com.facebook.presto.spi.type.TimeZoneKey; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.joda.time.DateTime; @@ -390,7 +389,7 @@ private InternalAggregationFunction getInternalDefaultVarCharAggregationn() private InternalAggregationFunction getAggregation(Type... arguments) { FunctionManager functionManager = getFunctionManager(NEW); - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(NAME, fromTypes(arguments))); } public FunctionManager getFunctionManager() diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java index e48abe2ec85e3..3ca29f7731fbc 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -189,6 +188,6 @@ public void testArrayDoubleMap() private InternalAggregationFunction getAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(NAME, fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapUnionAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapUnionAggregation.java index 34d3ef25549ff..d6c9526f3fe6c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapUnionAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapUnionAggregation.java @@ -18,7 +18,6 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.MapType; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -46,7 +45,7 @@ public class TestMapUnionAggregation public void testSimpleWithDuplicates() { MapType mapType = mapType(DOUBLE, VARCHAR); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + FunctionHandle functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); InternalAggregationFunction aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -57,7 +56,7 @@ public void testSimpleWithDuplicates() mapBlockOf(DOUBLE, VARCHAR, ImmutableMap.of(43.0, "ccc", 53.0, "ddd", 13.0, "eee")))); mapType = mapType(DOUBLE, BIGINT); - functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -68,7 +67,7 @@ public void testSimpleWithDuplicates() mapBlockOf(DOUBLE, BIGINT, ImmutableMap.of(1.0, 44L, 2.0, 44L, 4.0, 44L)))); mapType = mapType(BOOLEAN, BIGINT); - functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -83,7 +82,7 @@ public void testSimpleWithDuplicates() public void testSimpleWithNulls() { MapType mapType = mapType(DOUBLE, VARCHAR); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + FunctionHandle functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); InternalAggregationFunction aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); Map expected = mapOf(23.0, "aaa", 33.0, null, 43.0, "ccc", 53.0, "ddd"); @@ -102,7 +101,7 @@ public void testSimpleWithNulls() public void testStructural() { MapType mapType = mapType(DOUBLE, new ArrayType(VARCHAR)); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + FunctionHandle functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); InternalAggregationFunction aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -135,7 +134,7 @@ public void testStructural() ImmutableList.of("w", "z"))))); mapType = mapType(DOUBLE, mapType(VARCHAR, VARCHAR)); - functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -161,7 +160,7 @@ public void testStructural() ImmutableMap.of("e", "f"))))); mapType = mapType(new ArrayType(VARCHAR), DOUBLE); - functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMultimapAggAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMultimapAggAggregation.java index b4bf501f4e264..d750e62a8ae00 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMultimapAggAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMultimapAggAggregation.java @@ -25,7 +25,6 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; @@ -182,7 +181,7 @@ private static void testMultimapAgg(Type keyType, List expectedKeys, T private static InternalAggregationFunction getInternalAggregationFunction(Type keyType, Type valueType) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(keyType, valueType))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(NAME, fromTypes(keyType, valueType))); } private static void testMultimapAgg(InternalAggregationFunction aggFunc, Type keyType, List expectedKeys, Type valueType, List expectedValues) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java index b9e4f52f3a586..81c787ec488d2 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Joiner; import com.google.common.primitives.Floats; import io.airlift.stats.QuantileDigest; @@ -168,7 +167,7 @@ private InternalAggregationFunction getAggregationFunction(Type... type) { FunctionManager functionManager = METADATA.getFunctionManager(); return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("qdigest_agg"), fromTypes(type))); + functionManager.lookupFunction("qdigest_agg", fromTypes(type))); } private void testAggregationBigint(Block inputBlock, Block weightsBlock, double maxError, long... inputs) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestRealHistogramAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestRealHistogramAggregation.java index 9a18187389115..51ba0025248f3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestRealHistogramAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestRealHistogramAggregation.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -50,7 +49,7 @@ public TestRealHistogramAggregation() TypeRegistry typeRegistry = new TypeRegistry(); FunctionManager functionManager = new FunctionManager(typeRegistry, new BlockEncodingManager(typeRegistry), new FeaturesConfig()); InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("numeric_histogram"), fromTypes(BIGINT, REAL, DOUBLE))); + functionManager.lookupFunction("numeric_histogram", fromTypes(BIGINT, REAL, DOUBLE))); factory = function.bind(ImmutableList.of(0, 1, 2), Optional.empty()); input = makeInput(10); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByAggregation.java index 978e6cfb04c66..e9c088e16d74f 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByAggregation.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.SqlDecimal; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -578,11 +577,11 @@ public void testMaxUnknownLongArray() private InternalAggregationFunction getMinByAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("min_by"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("min_by", fromTypes(arguments))); } private InternalAggregationFunction getMaxByAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("max_by"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("max_by", fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByNAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByNAggregation.java index 2f008a1868b1a..939ce8f552e43 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByNAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByNAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -297,12 +296,12 @@ public void testOutOfBound() private InternalAggregationFunction getMaxByAggregation(Type... arguments) { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("max_by"), fromTypes(arguments))); + functionManager.lookupFunction("max_by", fromTypes(arguments))); } private InternalAggregationFunction getMinByAggregation(Type... arguments) { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("min_by"), fromTypes(arguments))); + functionManager.lookupFunction("min_by", fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java index dd218329b80d9..2fbe911217918 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java @@ -30,7 +30,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; @@ -115,7 +114,7 @@ public void setup() for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arrayType)); + FunctionHandle functionHandle = functionManager.lookupFunction(name, fromTypes(arrayType)); projectionsBuilder.add(new CallExpression(name, functionHandle, arrayType, ImmutableList.of(field(i, arrayType)))); blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java index b6d744629c1b5..f082be7c6c4af 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java @@ -35,7 +35,6 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import org.openjdk.jmh.annotations.Benchmark; @@ -132,7 +131,7 @@ public void setup() for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(name), fromTypeSignatures(arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)"))); + FunctionHandle functionHandle = functionManager.lookupFunction(name, fromTypeSignatures(arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)"))); FunctionHandle greaterThan = functionManager.resolveOperator(GREATER_THAN, fromTypes(BIGINT, BIGINT)); projectionsBuilder.add(new CallExpression(name, functionHandle, arrayType, ImmutableList.of( field(0, arrayType), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java index 621fc503b3481..85a45cc88498d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java @@ -34,7 +34,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -142,7 +141,7 @@ public void setup() throw new UnsupportedOperationException(); } ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arrayType)); + FunctionHandle functionHandle = functionManager.lookupFunction(name, fromTypes(arrayType)); projectionsBuilder.add(new CallExpression(name, functionHandle, BIGINT, ImmutableList.of(field(0, arrayType)))); blocks[0] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java index 56ca417cf0f79..6bb2a72a00444 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java @@ -27,7 +27,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -124,7 +123,7 @@ public void setup() ArrayType arrayType = new ArrayType(elementType); MetadataManager metadata = createTestMetadataManager(); FunctionManager functionManager = metadata.getFunctionManager(); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arrayType, arrayType)); + FunctionHandle functionHandle = functionManager.lookupFunction(name, fromTypes(arrayType, arrayType)); ImmutableList projections = ImmutableList.of( new CallExpression(name, functionHandle, arrayType, ImmutableList.of(field(0, arrayType), field(1, arrayType)))); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java index 7072dd202a55a..9d8d63fc28f7d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java @@ -26,7 +26,6 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -93,7 +92,7 @@ public void setup() { MetadataManager metadata = createTestMetadataManager(); FunctionManager functionManager = metadata.getFunctionManager(); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of("array_join"), fromTypes(new ArrayType(BIGINT), VARCHAR)); + FunctionHandle functionHandle = functionManager.lookupFunction("array_join", fromTypes(new ArrayType(BIGINT), VARCHAR)); List projections = ImmutableList.of( new CallExpression("array_join", functionHandle, VARCHAR, ImmutableList.of( diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java index d704645a01bdd..43567151df376 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java @@ -28,7 +28,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; @@ -113,7 +112,7 @@ public void setup() for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(QualifiedName.of(name), fromTypes(arrayType)); + FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(name, fromTypes(arrayType)); projectionsBuilder.add(new CallExpression(name, functionHandle, arrayType, ImmutableList.of(field(i, arrayType)))); blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java index 381f183bf66b4..dd4d013aa30eb 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java @@ -32,7 +32,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import org.openjdk.jmh.annotations.Benchmark; @@ -116,7 +115,7 @@ public void setup() for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of("transform"), fromTypeSignatures(arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)"))); + FunctionHandle functionHandle = functionManager.lookupFunction("transform", fromTypeSignatures(arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)"))); FunctionHandle greaterThan = functionManager.resolveOperator(GREATER_THAN, fromTypes(BIGINT, BIGINT)); projectionsBuilder.add(new CallExpression("transform", functionHandle, returnType, ImmutableList.of( new InputReferenceExpression(0, arrayType), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java index 5b96a58a758be..e839ec1e85531 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java @@ -26,7 +26,6 @@ import com.facebook.presto.spi.type.MapType; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import org.openjdk.jmh.annotations.Benchmark; @@ -139,7 +138,7 @@ public void setup() ImmutableList.Builder projectionsBuilder = ImmutableList.builder(); - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(QualifiedName.of(name), fromTypes(mapType, mapType)); + FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(name, fromTypes(mapType, mapType)); projectionsBuilder.add(new CallExpression( name, functionHandle, diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java index a56d9c657b7fd..9731a58881d76 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java @@ -29,7 +29,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -127,7 +126,7 @@ public void setup() } MapType mapType = mapType(elementType, elementType); FunctionHandle functionHandle = functionManager.lookupFunction( - QualifiedName.of(name), + name, fromTypeSignatures( mapType.getTypeSignature(), parseTypeSignature(format("function(%s, %s, %s)", type, type, type)))); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java index 29028f3bef456..acf75f4e0b84d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java @@ -29,7 +29,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -135,7 +134,7 @@ public void setup() MapType mapType = mapType(elementType, elementType); MapType returnType = mapType(elementType, BOOLEAN); FunctionHandle functionHandle = functionManager.lookupFunction( - QualifiedName.of(name), + name, fromTypeSignatures( mapType.getTypeSignature(), parseTypeSignature(format("function(%s, %s, boolean)", type, type)))); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java index 978cb78620450..bc5bcf41a408e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java @@ -28,7 +28,6 @@ import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; import com.facebook.presto.sql.relational.DeterminismEvaluator; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; @@ -83,7 +82,7 @@ public void testNoCaching() FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); ImmutableList.Builder projectionsBuilder = ImmutableList.builder(); ArrayType arrayType = new ArrayType(VARCHAR); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of("concat"), fromTypes(arrayType, arrayType)); + FunctionHandle functionHandle = functionManager.lookupFunction("concat", fromTypes(arrayType, arrayType)); projectionsBuilder.add(new CallExpression("concat", functionHandle, arrayType, ImmutableList.of(field(0, arrayType), field(1, arrayType)))); ImmutableList projections = projectionsBuilder.build(); @@ -123,7 +122,7 @@ public void testSanityFilterOnDictionary() { FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); CallExpression lengthVarchar = new CallExpression( - "length", functionManager.lookupFunction(QualifiedName.of("length"), fromTypes(VARCHAR)), BIGINT, ImmutableList.of(field(0, VARCHAR))); + "length", functionManager.lookupFunction("length", fromTypes(VARCHAR)), BIGINT, ImmutableList.of(field(0, VARCHAR))); FunctionHandle lessThan = functionManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT)); CallExpression filter = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(lengthVarchar, constant(10L, BIGINT))); @@ -211,7 +210,7 @@ public void testNonDeterministicProject() FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); FunctionHandle lessThan = functionManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT)); CallExpression random = new CallExpression( - "random", functionManager.lookupFunction(QualifiedName.of("random"), fromTypes(BIGINT)), BIGINT, singletonList(constant(10L, BIGINT))); + "random", functionManager.lookupFunction("random", fromTypes(BIGINT)), BIGINT, singletonList(constant(10L, BIGINT))); InputReferenceExpression col0 = field(0, BIGINT); CallExpression lessThanRandomExpression = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(col0, random)); diff --git a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java index ded91e9987bda..b697c77e17d62 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java +++ b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java @@ -30,7 +30,7 @@ import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.metadata.HandleResolver; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.server.TaskUpdateRequest; @@ -211,7 +211,7 @@ private RemoteTask createRemoteTask(HttpRemoteTaskFactory httpRemoteTaskFactory) return httpRemoteTaskFactory.createRemoteTask( TEST_SESSION, new TaskId("test", 1, 2), - new PrestoNode("node-id", URI.create("http://fake.invalid/"), new NodeVersion("version"), false), + new InternalNode("node-id", URI.create("http://fake.invalid/"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), OptionalInt.empty(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java index fba62a6b1f0e4..533201d145d25 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java @@ -89,6 +89,7 @@ import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionInterpreter; import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionOptimizer; import static com.facebook.presto.sql.planner.RowExpressionInterpreter.rowExpressionInterpreter; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.util.DateTimeZoneIndex.getDateTimeZone; import static io.airlift.slice.Slices.utf8Slice; @@ -352,6 +353,9 @@ public void testFunctionCall() assertOptimizedEquals("abs(-bound_long)", "1234"); assertOptimizedEquals("abs(unbound_long)", "abs(unbound_long)"); assertOptimizedEquals("abs(unbound_long + 1)", "abs(unbound_long + 1)"); + assertOptimizedEquals("cast(json_parse(unbound_string) as map(varchar, varchar))", "cast(json_parse(unbound_string) as map(varchar, varchar))"); + assertOptimizedEquals("cast(json_parse(unbound_string) as array(varchar))", "cast(json_parse(unbound_string) as array(varchar))"); + assertOptimizedEquals("cast(json_parse(unbound_string) as row(bigint, varchar))", "cast(json_parse(unbound_string) as row(bigint, varchar))"); } @Test @@ -1530,7 +1534,7 @@ private static Object optimize(@Language("SQL") String expression) return value; }); RowExpression rowExpression = TRANSLATOR.translate(parsedExpression, SYMBOL_TYPES); - Object rowExpressionResult = new RowExpressionInterpreter(rowExpression, METADATA, TEST_SESSION, true).optimize(symbol -> { + Object rowExpressionResult = new RowExpressionInterpreter(rowExpression, METADATA, TEST_SESSION.toConnectorSession(), true).optimize(symbol -> { Object value = symbolConstant(symbol); if (value == null) { return new VariableReferenceExpression(symbol.getName(), SYMBOL_TYPES.get(symbol)); @@ -1582,7 +1586,7 @@ private static void assertExpressionAndRowExpressionEquals(Object expressionResu // It is tricky to check the equivalence of an expression and a row expression. // We rely on the optimized translator to fill the gap. RowExpression translated = TRANSLATOR.translateAndOptimize((Expression) expressionResult, SYMBOL_TYPES); - assertRowExpressionEvaluationEquals(translated, new ExpressionOptimizer(METADATA.getFunctionManager(), TEST_SESSION).optimize((RowExpression) rowExpressionResult)); + assertRowExpressionEvaluationEquals(translated, new ExpressionOptimizer(METADATA.getFunctionManager(), SESSION).optimize((RowExpression) rowExpressionResult)); } else { // We have constants; directly compare @@ -1675,7 +1679,7 @@ private static Object evaluate(Expression expression, boolean deterministic) { Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, METADATA, SQL_PARSER, SYMBOL_TYPES, expression, emptyList(), WarningCollector.NOOP); Object expressionResult = expressionInterpreter(expression, METADATA, TEST_SESSION, expressionTypes).evaluate(); - Object rowExpressionResult = rowExpressionInterpreter(TRANSLATOR.translateAndOptimize(expression), METADATA, TEST_SESSION).evaluate(); + Object rowExpressionResult = rowExpressionInterpreter(TRANSLATOR.translateAndOptimize(expression), METADATA, TEST_SESSION.toConnectorSession()).evaluate(); if (deterministic) { assertExpressionAndRowExpressionEquals(expressionResult, rowExpressionResult); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java index 514dc239f90f7..ec1284645f325 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java @@ -26,14 +26,12 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.relational.optimizer.ExpressionOptimizer; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.toValues; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.metadata.CastType.JSON_TO_ARRAY_CAST; @@ -51,6 +49,7 @@ import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.field; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.facebook.presto.type.JsonType.JSON; import static com.facebook.presto.util.StructuralTestUtil.mapType; import static io.airlift.slice.Slices.utf8Slice; @@ -67,7 +66,7 @@ public void setUp() { TypeManager typeManager = new TypeRegistry(); functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - optimizer = new ExpressionOptimizer(functionManager, TEST_SESSION); + optimizer = new ExpressionOptimizer(functionManager, SESSION); } @AfterClass(alwaysRun = true) @@ -102,7 +101,7 @@ public void testIfConstantOptimization() @Test public void testCastWithJsonParseOptimization() { - FunctionHandle jsonParseFunctionHandle = functionManager.lookupFunction(QualifiedName.of("json_parse"), fromTypes(VARCHAR)); + FunctionHandle jsonParseFunctionHandle = functionManager.lookupFunction("json_parse", fromTypes(VARCHAR)); // constant FunctionHandle jsonCastFunctionHandle = functionManager.lookupCast(CAST, JSON.getTypeSignature(), parseTypeSignature("array(integer)")); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java b/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java index 362e634585137..4bda1a3160a1a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java @@ -16,6 +16,7 @@ import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.block.BlockJsonSerde; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.block.Block; @@ -39,7 +40,6 @@ import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NodeRef; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeDeserializer; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; @@ -233,7 +233,7 @@ private FunctionHandle operator(OperatorType operatorType, Type... types) private FunctionHandle function(String name, Type... types) { - return metadata.getFunctionManager().lookupFunction(QualifiedName.of(name), fromTypes(types)); + return metadata.getFunctionManager().lookupFunction(name, fromTypes(types)); } private JsonCodec getJsonCodec() @@ -241,6 +241,7 @@ private JsonCodec getJsonCodec() { Module module = binder -> { binder.install(new JsonModule()); + binder.install(new HandleJsonModule()); configBinder(binder).bindConfig(FeaturesConfig.class); binder.bind(TypeManager.class).to(TypeRegistry.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java index 80bf696a03e70..6378fe851be2b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java @@ -140,7 +140,7 @@ public void setUp() public void testAggregation() { FunctionCall functionCall = new FunctionCall(QualifiedName.of("count"), ImmutableList.of()); - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(QualifiedName.of("count"), ImmutableList.of()); + FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction("count", ImmutableList.of()); PlanNode node = new AggregationNode(newId(), filter(baseTableScan, and( diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java index 15ed0fdc74ac1..491889541bb2b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java @@ -21,16 +21,15 @@ import com.facebook.presto.spi.predicate.Range; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.predicate.ValueSet; +import com.facebook.presto.spi.relation.DomainTranslator.ExtractionResult; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; -import com.facebook.presto.sql.relational.RowExpressionDomainTranslator.ExtractionResult; -import com.facebook.presto.sql.relational.StandardFunctionResolution; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.BaseEncoding; @@ -52,6 +51,7 @@ import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; import static com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains; +import static com.facebook.presto.spi.relation.DomainTranslator.BASIC_COLUMN_EXTRACTOR; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IN; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -129,7 +129,7 @@ public class TestRowExpressionDomainTranslator public void setup() { metadata = createTestMetadataManager(); - domainTranslator = new RowExpressionDomainTranslator(metadata.getFunctionManager()); + domainTranslator = new RowExpressionDomainTranslator(metadata); } @AfterClass(alwaysRun = true) @@ -902,7 +902,7 @@ public void testFromNullLiteralPredicate() @Test public void testExpressionConstantFolding() { - FunctionHandle hex = metadata.getFunctionManager().lookupFunction(QualifiedName.of("from_hex"), fromTypes(VARCHAR)); + FunctionHandle hex = metadata.getFunctionManager().lookupFunction("from_hex", fromTypes(VARCHAR)); RowExpression originalExpression = greaterThan(C_VARBINARY, call("from_hex", hex, VARBINARY, stringLiteral("123456"))); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE); @@ -1076,6 +1076,7 @@ private void testNumericTypeTranslation(NumericValues columnValues, NumericValue public void testLegacyCharComparedToVarcharExpression() { metadata = createTestMetadataManager(new FeaturesConfig().setLegacyCharToVarcharCoercion(true)); + domainTranslator = new RowExpressionDomainTranslator(metadata); String maxCodePoint = new String(Character.toChars(Character.MAX_CODE_POINT)); @@ -1155,7 +1156,7 @@ private RowExpression toPredicate(TupleDomain tuple private ExtractionResult fromPredicate(RowExpression originalPredicate) { - return RowExpressionDomainTranslator.fromPredicate(metadata, TEST_SESSION, originalPredicate); + return domainTranslator.fromPredicate(TEST_SESSION.toConnectorSession(), originalPredicate, BASIC_COLUMN_EXTRACTOR); } private RowExpression nullLiteral(Type type) @@ -1221,7 +1222,7 @@ private RowExpression cast(RowExpression expression, Type toType) private RowExpression not(RowExpression expression) { - return call("not", new StandardFunctionResolution(metadata.getFunctionManager()).notFunction(), expression.getType(), expression); + return call("not", new FunctionResolution(metadata.getFunctionManager()).notFunction(), expression.getType(), expression); } private RowExpression in(RowExpression value, List inList) @@ -1308,7 +1309,7 @@ private RowExpression unprocessableExpression2(VariableReferenceExpression expre private RowExpression randPredicate(VariableReferenceExpression expression) { - RowExpression random = call("random", metadata.getFunctionManager().lookupFunction(QualifiedName.of("random"), fromTypes()), DOUBLE); + RowExpression random = call("random", metadata.getFunctionManager().lookupFunction("random", fromTypes()), DOUBLE); return greaterThan( expression, call(CastType.CAST.name(), metadata.getFunctionManager().lookupCast(CastType.CAST, DOUBLE.getTypeSignature(), expression.getType().getTypeSignature()), expression.getType(), random)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionRewriter.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionRewriter.java new file mode 100644 index 0000000000000..dc4fde02765eb --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionRewriter.java @@ -0,0 +1,139 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionRewriter; +import com.facebook.presto.sql.relational.RowExpressionTreeRewriter; +import com.facebook.presto.type.TypeRegistry; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.function.OperatorType.ADD; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.planner.TestRowExpressionRewriter.NegationExpressionRewriter.rewrite; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.RowExpressionNodeInliner.replaceExpression; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestRowExpressionRewriter +{ + private FunctionManager functionManager; + + @BeforeClass + public void setup() + { + TypeRegistry typeManager = new TypeRegistry(); + this.functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + } + + @Test + public void testSimple() + { + // successful rewrite + RowExpression predicate = call( + GREATER_THAN.name(), + functionManager.resolveOperator(GREATER_THAN, fromTypes(BIGINT, BIGINT)), + BOOLEAN, + constant(1L, BIGINT), + constant(2L, BIGINT)); + + RowExpression negatedPredicate = rewrite(predicate); + assertEquals(negatedPredicate.getType(), BOOLEAN); + assertTrue(negatedPredicate instanceof CallExpression); + assertTrue(((CallExpression) negatedPredicate).getArguments().get(0) instanceof CallExpression); + assertEquals(((CallExpression) negatedPredicate).getDisplayName(), "not"); + assertEquals(((CallExpression) ((CallExpression) negatedPredicate).getArguments().get(0)).getDisplayName(), GREATER_THAN.name()); + + // no rewrite + RowExpression nonPredicate = call( + ADD.name(), + functionManager.resolveOperator(ADD, fromTypes(BIGINT, BIGINT)), + BIGINT, + constant(1L, BIGINT), + constant(2L, BIGINT)); + + RowExpression samePredicate = rewrite(nonPredicate); + assertEquals(samePredicate, nonPredicate); + } + + @Test + public void testInliner() + { + RowExpression predicate = call( + GREATER_THAN.name(), + functionManager.resolveOperator(GREATER_THAN, fromTypes(BIGINT, BIGINT)), + BOOLEAN, + constant(1L, BIGINT), + constant(2L, BIGINT)); + + // no rewrite + RowExpression nonPredicate = call( + ADD.name(), + functionManager.resolveOperator(ADD, fromTypes(BIGINT, BIGINT)), + BIGINT, + constant(1L, BIGINT), + constant(2L, BIGINT)); + + RowExpression samePredicate = replaceExpression(predicate, ImmutableMap.of(predicate, nonPredicate)); + assertEquals(samePredicate, nonPredicate); + } + + public static class NegationExpressionRewriter + { + private NegationExpressionRewriter() {} + + public static RowExpression rewrite(RowExpression expression) + { + return RowExpressionTreeRewriter.rewriteWith(new Visitor(), expression); + } + + private static class Visitor + extends RowExpressionRewriter + { + private final FunctionManager functionManager; + private final StandardFunctionResolution functionResolution; + + Visitor() + { + TypeRegistry typeManager = new TypeRegistry(); + this.functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + this.functionResolution = new FunctionResolution(functionManager); + } + + @Override + public RowExpression rewriteCall(CallExpression node, Void context, RowExpressionTreeRewriter treeRewriter) + { + FunctionMetadata metadata = functionManager.getFunctionMetadata(node.getFunctionHandle()); + if (metadata.getOperatorType().isPresent() && metadata.getOperatorType().get().isComparisonOperator()) { + return call("not", functionResolution.notFunction(), BOOLEAN, node); + } + return null; + } + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java index dd92f7c32200d..ec32e38d3aa9c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java @@ -148,7 +148,7 @@ public void testValidUnion() public void testValidWindow() { Symbol windowSymbol = symbolAllocator.newSymbol("sum", DOUBLE); - FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)); + FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction("sum", fromTypes(DOUBLE)); FunctionCall functionCall = new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())); WindowNode.Frame frame = new WindowNode.Frame( @@ -186,7 +186,7 @@ public void testValidAggregation() baseTableScan, ImmutableMap.of(aggregationSymbol, new Aggregation( new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())), - FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)), + FUNCTION_MANAGER.lookupFunction("sum", fromTypes(DOUBLE)), Optional.empty())), singleGroupingSet(ImmutableList.of(columnA, columnB)), ImmutableList.of(), @@ -237,7 +237,7 @@ public void testInvalidAggregationFunctionCall() baseTableScan, ImmutableMap.of(aggregationSymbol, new Aggregation( new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnA.toSymbolReference())), - FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)), + FUNCTION_MANAGER.lookupFunction("sum", fromTypes(DOUBLE)), Optional.empty())), singleGroupingSet(ImmutableList.of(columnA, columnB)), ImmutableList.of(), @@ -258,7 +258,7 @@ public void testInvalidAggregationFunctionSignature() baseTableScan, ImmutableMap.of(aggregationSymbol, new Aggregation( new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())), - FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT)), // should be DOUBLE + FUNCTION_MANAGER.lookupFunction("sum", fromTypes(BIGINT)), // should be DOUBLE Optional.empty())), singleGroupingSet(ImmutableList.of(columnA, columnB)), ImmutableList.of(), @@ -273,7 +273,7 @@ public void testInvalidAggregationFunctionSignature() public void testInvalidWindowFunctionCall() { Symbol windowSymbol = symbolAllocator.newSymbol("sum", DOUBLE); - FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)); + FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction("sum", fromTypes(DOUBLE)); FunctionCall functionCall = new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnA.toSymbolReference())); // should be columnC WindowNode.Frame frame = new WindowNode.Frame( @@ -305,7 +305,7 @@ public void testInvalidWindowFunctionCall() public void testInvalidWindowFunctionSignature() { Symbol windowSymbol = symbolAllocator.newSymbol("sum", DOUBLE); - FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT)); // should be DOUBLE + FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction("sum", fromTypes(BIGINT)); // should be DOUBLE FunctionCall functionCall = new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())); WindowNode.Frame frame = new WindowNode.Frame( diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java index aff32bf762e2d..c0d733ba538a2 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java @@ -14,8 +14,8 @@ package com.facebook.presto.sql.planner.assertions; import com.facebook.presto.Session; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; @@ -23,7 +23,7 @@ import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.LiteralInterpreter; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.AstVisitor; import com.facebook.presto.sql.tree.BetweenPredicate; @@ -91,14 +91,14 @@ final class RowExpressionVerifier private final SymbolAliases symbolAliases; private final Metadata metadata; private final Session session; - private final StandardFunctionResolution functionResolution; + private final FunctionResolution functionResolution; RowExpressionVerifier(SymbolAliases symbolAliases, Metadata metadata, Session session) { this.symbolAliases = requireNonNull(symbolAliases, "symbolLayout is null"); this.metadata = requireNonNull(metadata, "metadata is null"); this.session = requireNonNull(session, "session is null"); - this.functionResolution = new StandardFunctionResolution(metadata.getFunctionManager()); + this.functionResolution = new FunctionResolution(metadata.getFunctionManager()); } @Override @@ -110,7 +110,7 @@ protected Boolean visitNode(Node node, RowExpression context) @Override protected Boolean visitTryExpression(TryExpression expected, RowExpression actual) { - if (!(actual instanceof CallExpression) || !((CallExpression) actual).getFunctionHandle().getSignature().getName().equals("TRY")) { + if (!(actual instanceof CallExpression) || !functionResolution.isTryFunction(((CallExpression) actual).getFunctionHandle())) { return false; } @@ -343,7 +343,7 @@ else if (expression instanceof GenericLiteral) { private Boolean compareLiteral(Node expected, RowExpression actual) { if (actual instanceof CallExpression && functionResolution.isCastFunction(((CallExpression) actual).getFunctionHandle())) { - return getValueFromLiteral(expected).equals(String.valueOf(rowExpressionInterpreter(actual, metadata, session).evaluate())); + return getValueFromLiteral(expected).equals(String.valueOf(rowExpressionInterpreter(actual, metadata, session.toConnectorSession()).evaluate())); } if (actual instanceof ConstantExpression) { return getValueFromLiteral(expected).equals(String.valueOf(LiteralInterpreter.evaluate(TEST_SESSION.toConnectorSession(), (ConstantExpression) actual))); @@ -473,7 +473,7 @@ protected Boolean visitFunctionCall(FunctionCall expected, RowExpression actual) } CallExpression actualFunction = (CallExpression) actual; - if (!expected.getName().equals(QualifiedName.of(actualFunction.getFunctionHandle().getSignature().getName()))) { + if (!expected.getName().equals(QualifiedName.of(metadata.getFunctionManager().getFunctionMetadata(actualFunction.getFunctionHandle()).getName()))) { return false; } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java index a941a0d1569da..8c9fd78c911fa 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java @@ -58,7 +58,7 @@ public class TestMergeAdjacentWindows Optional.empty(), Optional.empty()); - private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().lookupFunction(QualifiedName.of("avg"), fromTypes(DOUBLE)); + private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().lookupFunction("avg", fromTypes(DOUBLE)); private static final String columnAAlias = "ALIAS_A"; private static final ExpectedValueProvider specificationA = specification(ImmutableList.of(columnAAlias), ImmutableList.of(), ImmutableMap.of()); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java index 69cc6fc72468b..8427deb005342 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java @@ -39,7 +39,6 @@ import java.util.Set; import java.util.function.Predicate; -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; @@ -58,7 +57,7 @@ public class TestPruneWindowColumns extends BaseRuleTest { - private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().resolveFunction(TEST_SESSION, QualifiedName.of("min"), fromTypes(BIGINT)); + private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().lookupFunction("min", fromTypes(BIGINT)); private static final List inputSymbolNameList = ImmutableList.of("orderKey", "partitionKey", "hash", "startValue1", "startValue2", "endValue1", "endValue2", "input1", "input2", "unused"); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java index a895a29dc56a4..e567d6de37f22 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java @@ -56,7 +56,7 @@ public TestSwapAdjacentWindowsBySpecifications() Optional.empty(), Optional.empty()); - functionHandle = createTestMetadataManager().getFunctionManager().lookupFunction(QualifiedName.of("avg"), fromTypes(BIGINT)); + functionHandle = createTestMetadataManager().getFunctionManager().lookupFunction("avg", fromTypes(BIGINT)); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java index 2f3b3b45f4d2e..ce9eeef29bac9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java @@ -361,8 +361,7 @@ protected AggregationNode build() public ApplyNode apply(Assignments subqueryAssignments, List correlation, PlanNode input, PlanNode subquery) { - NullLiteral originSubquery = new NullLiteral(); // does not matter for tests - return new ApplyNode(idAllocator.getNextId(), input, subquery, subqueryAssignments, correlation, originSubquery); + return new ApplyNode(idAllocator.getNextId(), input, subquery, subqueryAssignments, correlation, ""); } public AssignUniqueId assignUniqueId(Symbol unique, PlanNode source) @@ -373,7 +372,7 @@ public AssignUniqueId assignUniqueId(Symbol unique, PlanNode source) public LateralJoinNode lateral(List correlation, PlanNode input, PlanNode subquery) { NullLiteral originSubquery = new NullLiteral(); // does not matter for tests - return new LateralJoinNode(idAllocator.getNextId(), input, subquery, correlation, LateralJoinNode.Type.INNER, originSubquery); + return new LateralJoinNode(idAllocator.getNextId(), input, subquery, correlation, LateralJoinNode.Type.INNER, ""); } public TableScanNode tableScan(List symbols, Map assignments) @@ -761,6 +760,7 @@ public UnnestNode unnest(PlanNode source, List replicateSymbols, Map codec; public TestWindowNode() + throws Exception { - // dependencies copied from ServerMainModule.java to avoid depending on whole ServerMainModule here - SqlParser sqlParser = new SqlParser(); - ObjectMapperProvider provider = new ObjectMapperProvider(); - provider.setJsonSerializers(ImmutableMap.of( - Slice.class, new SliceSerializer(), - Expression.class, new Serialization.ExpressionSerializer())); - provider.setJsonDeserializers(ImmutableMap.of( - Slice.class, new SliceDeserializer(), - Expression.class, new Serialization.ExpressionDeserializer(sqlParser), - FunctionCall.class, new Serialization.FunctionCallDeserializer(sqlParser))); - objectMapper = provider.get(); + codec = getJsonCodec(); } @BeforeClass @@ -91,7 +89,7 @@ public void testSerializationRoundtrip() throws Exception { Symbol windowSymbol = symbolAllocator.newSymbol("sum", BIGINT); - FunctionHandle functionHandle = createTestMetadataManager().getFunctionManager().lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT)); + FunctionHandle functionHandle = createTestMetadataManager().getFunctionManager().lookupFunction("sum", fromTypes(BIGINT)); FunctionCall functionCall = new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())); WindowNode.Frame frame = new WindowNode.Frame( RANGE, @@ -120,9 +118,9 @@ public void testSerializationRoundtrip() prePartitionedInputs, 0); - String json = objectMapper.writeValueAsString(windowNode); + String json = codec.toJson(windowNode); - WindowNode actualNode = objectMapper.readValue(json, WindowNode.class); + WindowNode actualNode = codec.fromJson(json); assertEquals(actualNode.getId(), windowNode.getId()); assertEquals(actualNode.getSpecification(), windowNode.getSpecification()); @@ -137,4 +135,28 @@ private static PlanNodeId newId() { return new PlanNodeId(UUID.randomUUID().toString()); } + + private JsonCodec getJsonCodec() + throws Exception + { + Module module = binder -> { + SqlParser sqlParser = new SqlParser(); + binder.install(new JsonModule()); + binder.install(new HandleJsonModule()); + binder.bind(SqlParser.class).toInstance(sqlParser); + jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class); + jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class); + jsonBinder(binder).addSerializerBinding(Expression.class).to(Serialization.ExpressionSerializer.class); + jsonBinder(binder).addDeserializerBinding(Expression.class).to(Serialization.ExpressionDeserializer.class); + jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(Serialization.FunctionCallDeserializer.class); + jsonCodecBinder(binder).bindJsonCodec(WindowNode.class); + }; + Bootstrap app = new Bootstrap(ImmutableList.of(module)); + Injector injector = app + .strictConfig() + .doNotInitializeLogging() + .quiet() + .initialize(); + return injector.getInstance(new Key>() {}); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDeterminismEvaluator.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDeterminismEvaluator.java index 28a5b3b9f184a..f181754bdbdee 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDeterminismEvaluator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDeterminismEvaluator.java @@ -17,7 +17,6 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -42,7 +41,7 @@ public void testDeterminismEvaluator() CallExpression random = new CallExpression( "random", - functionManager.lookupFunction(QualifiedName.of("random"), fromTypes(BIGINT)), + functionManager.lookupFunction("random", fromTypes(BIGINT)), BIGINT, singletonList(constant(10L, BIGINT))); assertFalse(determinismEvaluator.isDeterministic(random)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestFunctionResolution.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestFunctionResolution.java new file mode 100644 index 0000000000000..4688e519d1a35 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestFunctionResolution.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.type.TypeRegistry; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.function.OperatorType.ADD; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestFunctionResolution +{ + private FunctionResolution functionResolution; + + @BeforeClass + public void setup() + { + TypeManager typeManager = new TypeRegistry(); + FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + functionResolution = new FunctionResolution(functionManager); + } + + @Test + public void testStandardFunctionResolution() + { + StandardFunctionResolution standardFunctionResolution = functionResolution; + + // not + assertTrue(standardFunctionResolution.isNotFunction(standardFunctionResolution.notFunction())); + + // negate + assertTrue(standardFunctionResolution.isNegateFunction(standardFunctionResolution.negateFunction(DOUBLE))); + assertFalse(standardFunctionResolution.isNotFunction(standardFunctionResolution.negateFunction(DOUBLE))); + + // arithmetic + assertTrue(standardFunctionResolution.isArithmeticFunction(standardFunctionResolution.arithmeticFunction(ADD, DOUBLE, DOUBLE))); + assertFalse(standardFunctionResolution.isComparisonFunction(standardFunctionResolution.arithmeticFunction(ADD, DOUBLE, DOUBLE))); + + // comparison + assertTrue(standardFunctionResolution.isComparisonFunction(standardFunctionResolution.comparisonFunction(GREATER_THAN, DOUBLE, DOUBLE))); + assertFalse(standardFunctionResolution.isArithmeticFunction(standardFunctionResolution.comparisonFunction(GREATER_THAN, DOUBLE, DOUBLE))); + + // between + assertTrue(standardFunctionResolution.isBetweenFunction(standardFunctionResolution.betweenFunction(DOUBLE, DOUBLE, DOUBLE))); + assertFalse(standardFunctionResolution.isNotFunction(standardFunctionResolution.betweenFunction(DOUBLE, DOUBLE, DOUBLE))); + + // subscript + assertTrue(standardFunctionResolution.isSubscriptFunction(standardFunctionResolution.subscriptFunction(new ArrayType(DOUBLE), BIGINT))); + assertFalse(standardFunctionResolution.isBetweenFunction(standardFunctionResolution.subscriptFunction(new ArrayType(DOUBLE), BIGINT))); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java index d415a1d4f36ce..05415005b2e4f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import org.testng.annotations.BeforeClass; @@ -112,8 +111,8 @@ public void testDeterminism() { RowExpression a = name("a"); RowExpression b = name("b"); - RowExpression nondeterministic = call("random", functionManager.lookupFunction(QualifiedName.of("random"), fromTypes()), DOUBLE); - RowExpression deterministic = call("length", functionManager.lookupFunction(QualifiedName.of("length"), fromTypes(VARCHAR)), INTEGER); + RowExpression nondeterministic = call("random", functionManager.lookupFunction("random", fromTypes()), DOUBLE); + RowExpression deterministic = call("length", functionManager.lookupFunction("length", fromTypes(VARCHAR)), INTEGER); RowExpression expression = and(and(a, or(b, nondeterministic)), deterministic); diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index e0201cd333cef..14699e39611f1 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -21,11 +21,6 @@ presto-array - - com.facebook.presto - presto-parser - - com.facebook.thirdparty libsvm diff --git a/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java b/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java index dec4e14fdf769..4f8b007a901e6 100644 --- a/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java +++ b/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -45,7 +44,7 @@ public void testEvaluateClassifierPredictions() { metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions())); InternalAggregationFunction aggregation = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("evaluate_classifier_predictions"), fromTypes(BIGINT, BIGINT))); + functionManager.lookupFunction("evaluate_classifier_predictions", fromTypes(BIGINT, BIGINT))); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator(); accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java index 6195a7f9ff931..9d16b73ebe162 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java @@ -17,6 +17,8 @@ import com.google.common.collect.ImmutableList; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; +import com.mongodb.Tag; +import com.mongodb.TagSet; import io.airlift.configuration.Config; import io.airlift.configuration.DefunctConfig; @@ -36,6 +38,8 @@ public class MongoClientConfig private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); private static final Splitter PORT_SPLITTER = Splitter.on(':').trimResults().omitEmptyStrings(); private static final Splitter USER_SPLITTER = Splitter.onPattern("[:@]").trimResults().omitEmptyStrings(); + private static final Splitter TAGSET_SPLITTER = Splitter.on('&').trimResults().omitEmptyStrings(); + private static final Splitter TAG_SPLITTER = Splitter.on(':').trimResults().omitEmptyStrings(); private String schemaCollection = "_schema"; private List seeds = ImmutableList.of(); @@ -53,6 +57,7 @@ public class MongoClientConfig private int cursorBatchSize; // use driver default private ReadPreferenceType readPreference = ReadPreferenceType.PRIMARY; + private List readPreferenceTagSets = ImmutableList.of(); private WriteConcernType writeConcern = WriteConcernType.ACKNOWLEDGED; private String requiredReplicaSetName; private String implicitRowFieldPrefix = "_pos"; @@ -226,6 +231,38 @@ public MongoClientConfig setReadPreference(ReadPreferenceType readPreference) return this; } + public List getReadPreferenceTags() + { + return readPreferenceTagSets; + } + + @Config("mongodb.read-preference-tags") + public MongoClientConfig setReadPreferenceTags(String readPreferenceTags) + { + this.readPreferenceTagSets = buildTagSets(TAGSET_SPLITTER.split(readPreferenceTags)); + return this; + } + + private List buildTagSets(Iterable tagSets) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (String tagSet : tagSets) { + builder.add(new TagSet(buildTags(SPLITTER.split(tagSet)))); + } + return builder.build(); + } + + private List buildTags(Iterable tags) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (String tag : tags) { + List values = TAG_SPLITTER.splitToList(tag); + checkArgument(values.size() == 2, "Invalid Tag format. Requires tagName:tagValue"); + builder.add(new Tag(values.get(0), values.get(1))); + } + return builder.build(); + } + public WriteConcernType getWriteConcern() { return writeConcern; diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java index b0f54dd5eae9a..58c7cae22da45 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java @@ -55,13 +55,19 @@ public static MongoSession createMongoSession(TypeManager typeManager, MongoClie .sslEnabled(config.getSslEnabled()) .maxWaitTime(config.getMaxWaitTime()) .minConnectionsPerHost(config.getMinConnectionsPerHost()) - .readPreference(config.getReadPreference().getReadPreference()) .writeConcern(config.getWriteConcern().getWriteConcern()); if (config.getRequiredReplicaSetName() != null) { options.requiredReplicaSetName(config.getRequiredReplicaSetName()); } + if (config.getReadPreferenceTags().isEmpty()) { + options.readPreference(config.getReadPreference().getReadPreference()); + } + else { + options.readPreference(config.getReadPreference().getReadPreferenceWithTags(config.getReadPreferenceTags())); + } + MongoClient client = new MongoClient(config.getSeeds(), config.getCredentials(), options.build()); return new MongoSession( diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/ReadPreferenceType.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/ReadPreferenceType.java index 0c779f6ff45f9..f1fbec0a6bd92 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/ReadPreferenceType.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/ReadPreferenceType.java @@ -13,8 +13,13 @@ */ package com.facebook.presto.mongodb; +import com.facebook.presto.spi.PrestoException; import com.mongodb.ReadPreference; +import com.mongodb.TagSet; +import java.util.List; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static java.util.Objects.requireNonNull; public enum ReadPreferenceType @@ -36,4 +41,13 @@ public ReadPreference getReadPreference() { return readPreference; } + + public ReadPreference getReadPreferenceWithTags(List tagSets) + { + if (PRIMARY.equals(this)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Primary read preference can not specify tag sets"); + } + + return ReadPreference.valueOf(readPreference.getName(), tagSets); + } } diff --git a/presto-product-tests/conf/docker/common/compose-commons.sh b/presto-product-tests/conf/docker/common/compose-commons.sh index 70c9c247f89c8..ce27ea949c399 100644 --- a/presto-product-tests/conf/docker/common/compose-commons.sh +++ b/presto-product-tests/conf/docker/common/compose-commons.sh @@ -33,7 +33,7 @@ export HADOOP_BASE_IMAGE=${HADOOP_BASE_IMAGE:-"prestodb/hdp2.6-hive"} if [[ -z "${PRESTO_SERVER_DIR:-}" ]]; then source "${PRODUCT_TESTS_ROOT}/target/classes/presto.env" - PRESTO_SERVER_DIR="${PROJECT_ROOT}/presto-server/target/presto-server-${PRESTO_VERSION}/" + PRESTO_SERVER_DIR="${PROJECT_ROOT}/presto-server/target/presto-server-${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}/" fi export_canonical_path PRESTO_SERVER_DIR diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java index 5a867b419c3cf..ecafb08057adb 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java @@ -36,7 +36,9 @@ public enum RaptorErrorCode RAPTOR_CORRUPT_METADATA(11, EXTERNAL), RAPTOR_LOCAL_DISK_FULL(12, EXTERNAL), RAPTOR_BACKUP_CORRUPTION(13, EXTERNAL), - RAPTOR_NOT_ENOUGH_NODES(14, EXTERNAL); + RAPTOR_NOT_ENOUGH_NODES(14, EXTERNAL), + RAPTOR_WRITER_DATA_ERROR(15, EXTERNAL), + RAPTOR_UNSUPPORTED_COMPRESSION_KIND(16, EXTERNAL); private final ErrorCode errorCode; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java index c89ceee8c5531..14772f806e9d1 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java @@ -75,6 +75,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongConsumer; +import java.util.stream.Collectors; import static com.facebook.presto.raptor.RaptorBucketFunction.validateBucketType; import static com.facebook.presto.raptor.RaptorColumnHandle.BUCKET_NUMBER_COLUMN_NAME; @@ -188,6 +189,7 @@ private RaptorTableHandle getTableHandle(SchemaTableName tableName) table.getBucketCount(), table.isOrganized(), OptionalLong.empty(), + Optional.empty(), false); } @@ -775,6 +777,8 @@ public ConnectorTableHandle beginDelete(ConnectorSession session, ConnectorTable setTransactionId(transactionId); + Map columnTypes = dao.listTableColumns(handle.getTableId()).stream().collect(Collectors.toMap(k -> String.valueOf(k.getColumnId()), TableColumn::getDataType)); + return new RaptorTableHandle( connectorId, handle.getSchemaName(), @@ -785,6 +789,7 @@ public ConnectorTableHandle beginDelete(ConnectorSession session, ConnectorTable handle.getBucketCount(), handle.isOrganized(), OptionalLong.of(transactionId), + Optional.of(columnTypes), true); } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java index c6ab4f4492ad8..4438787ffa394 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java @@ -29,6 +29,8 @@ import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.UUID; @@ -56,14 +58,15 @@ public ConnectorPageSource createPageSource(ConnectorTransactionHandle transacti TupleDomain predicate = raptorSplit.getEffectivePredicate(); ReaderAttributes attributes = ReaderAttributes.from(session); OptionalLong transactionId = raptorSplit.getTransactionId(); + Optional> columnTypes = raptorSplit.getColumnTypes(); if (raptorSplit.getShardUuids().size() == 1) { UUID shardUuid = raptorSplit.getShardUuids().iterator().next(); - return createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId); + return createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId, columnTypes); } Iterator iterator = raptorSplit.getShardUuids().stream() - .map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId)) + .map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId, columnTypes)) .iterator(); return new ConcatPageSource(iterator); @@ -75,12 +78,13 @@ private ConnectorPageSource createPageSource( List columns, TupleDomain predicate, ReaderAttributes attributes, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> allColumnTypes) { List columnHandles = columns.stream().map(RaptorColumnHandle.class::cast).collect(toList()); List columnIds = columnHandles.stream().map(RaptorColumnHandle::getColumnId).collect(toList()); List columnTypes = columnHandles.stream().map(RaptorColumnHandle::getColumnType).collect(toList()); - return storageManager.getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, predicate, attributes, transactionId); + return storageManager.getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, predicate, attributes, transactionId, allColumnTypes); } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplit.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplit.java index c0ad106e324b3..1a9433539f300 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplit.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplit.java @@ -16,12 +16,15 @@ import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; @@ -39,6 +42,7 @@ public class RaptorSplit private final List addresses; private final TupleDomain effectivePredicate; private final OptionalLong transactionId; + private final Optional> columnTypes; @JsonCreator public RaptorSplit( @@ -46,9 +50,10 @@ public RaptorSplit( @JsonProperty("shardUuids") Set shardUuids, @JsonProperty("bucketNumber") OptionalInt bucketNumber, @JsonProperty("effectivePredicate") TupleDomain effectivePredicate, - @JsonProperty("transactionId") OptionalLong transactionId) + @JsonProperty("transactionId") OptionalLong transactionId, + @JsonProperty("columnTypes") Optional> columnTypes) { - this(connectorId, shardUuids, bucketNumber, ImmutableList.of(), effectivePredicate, transactionId); + this(connectorId, shardUuids, bucketNumber, ImmutableList.of(), effectivePredicate, transactionId, columnTypes); } public RaptorSplit( @@ -56,9 +61,10 @@ public RaptorSplit( UUID shardUuid, List addresses, TupleDomain effectivePredicate, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> columnTypes) { - this(connectorId, ImmutableSet.of(shardUuid), OptionalInt.empty(), addresses, effectivePredicate, transactionId); + this(connectorId, ImmutableSet.of(shardUuid), OptionalInt.empty(), addresses, effectivePredicate, transactionId, columnTypes); } public RaptorSplit( @@ -67,9 +73,10 @@ public RaptorSplit( int bucketNumber, HostAddress address, TupleDomain effectivePredicate, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> columnTypes) { - this(connectorId, shardUuids, OptionalInt.of(bucketNumber), ImmutableList.of(address), effectivePredicate, transactionId); + this(connectorId, shardUuids, OptionalInt.of(bucketNumber), ImmutableList.of(address), effectivePredicate, transactionId, columnTypes); } private RaptorSplit( @@ -78,7 +85,8 @@ private RaptorSplit( OptionalInt bucketNumber, List addresses, TupleDomain effectivePredicate, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> columnTypes) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.shardUuids = ImmutableSet.copyOf(requireNonNull(shardUuids, "shardUuid is null")); @@ -86,6 +94,7 @@ private RaptorSplit( this.addresses = ImmutableList.copyOf(requireNonNull(addresses, "addresses is null")); this.effectivePredicate = requireNonNull(effectivePredicate, "effectivePredicate is null"); this.transactionId = requireNonNull(transactionId, "transactionId is null"); + this.columnTypes = requireNonNull(columnTypes, "columnTypes is null"); } @Override @@ -130,6 +139,12 @@ public OptionalLong getTransactionId() return transactionId; } + @JsonProperty + public Optional> getColumnTypes() + { + return columnTypes; + } + @Override public Object getInfo() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java index 0a3d0ad5b5884..720246302b50a 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; import org.skife.jdbi.v2.ResultIterator; @@ -106,7 +107,7 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, Co OptionalLong transactionId = table.getTransactionId(); Optional> bucketToNode = handle.getPartitioning().map(RaptorPartitioningHandle::getBucketToNode); verify(bucketed == bucketToNode.isPresent(), "mismatched bucketCount and bucketToNode presence"); - return new RaptorSplitSource(tableId, merged, effectivePredicate, transactionId, bucketToNode); + return new RaptorSplitSource(tableId, merged, effectivePredicate, transactionId, table.getColumnTypes(), bucketToNode); } private static List getAddressesForNodes(Map nodeMap, Iterable nodeIdentifiers) @@ -140,6 +141,7 @@ private class RaptorSplitSource private final long tableId; private final TupleDomain effectivePredicate; private final OptionalLong transactionId; + private final Optional> columnTypes; private final Optional> bucketToNode; private final ResultIterator iterator; @@ -151,11 +153,13 @@ public RaptorSplitSource( boolean merged, TupleDomain effectivePredicate, OptionalLong transactionId, + Optional> columnTypes, Optional> bucketToNode) { this.tableId = tableId; this.effectivePredicate = requireNonNull(effectivePredicate, "effectivePredicate is null"); this.transactionId = requireNonNull(transactionId, "transactionId is null"); + this.columnTypes = requireNonNull(columnTypes, "columnTypesis null"); this.bucketToNode = requireNonNull(bucketToNode, "bucketToNode is null"); ResultIterator iterator; @@ -238,7 +242,7 @@ private ConnectorSplit createSplit(BucketShards bucketShards) addresses = ImmutableList.of(node.getHostAndPort()); } - return new RaptorSplit(connectorId, shardId, addresses, effectivePredicate, transactionId); + return new RaptorSplit(connectorId, shardId, addresses, effectivePredicate, transactionId, columnTypes); } private ConnectorSplit createBucketSplit(int bucketNumber, Set shards) @@ -257,7 +261,7 @@ private ConnectorSplit createBucketSplit(int bucketNumber, Set shard .collect(toSet()); HostAddress address = node.getHostAndPort(); - return new RaptorSplit(connectorId, shardUuids, bucketNumber, address, effectivePredicate, transactionId); + return new RaptorSplit(connectorId, shardUuids, bucketNumber, address, effectivePredicate, transactionId, columnTypes); } } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java index 736f820896b55..97a4bd49e6966 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java @@ -14,9 +14,11 @@ package com.facebook.presto.raptor; import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; @@ -39,6 +41,7 @@ public final class RaptorTableHandle private final OptionalInt bucketCount; private final boolean organized; private final OptionalLong transactionId; + private final Optional> columnTypes; private final boolean delete; @JsonCreator @@ -52,6 +55,7 @@ public RaptorTableHandle( @JsonProperty("bucketCount") OptionalInt bucketCount, @JsonProperty("organized") boolean organized, @JsonProperty("transactionId") OptionalLong transactionId, + @JsonProperty("columnTypes") Optional> columnTypes, @JsonProperty("delete") boolean delete) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); @@ -66,6 +70,7 @@ public RaptorTableHandle( this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); this.organized = organized; this.transactionId = requireNonNull(transactionId, "transactionId is null"); + this.columnTypes = requireNonNull(columnTypes, "columnTypes is null"); this.delete = delete; } @@ -129,6 +134,12 @@ public OptionalLong getTransactionId() return transactionId; } + @JsonProperty + public Optional> getColumnTypes() + { + return columnTypes; + } + @JsonProperty public boolean isDelete() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ColumnStats.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ColumnStats.java index 85154159f6eb0..544e405d02b91 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ColumnStats.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ColumnStats.java @@ -18,6 +18,8 @@ import javax.annotation.Nullable; +import java.util.Objects; + import static com.google.common.base.MoreObjects.toStringHelper; public class ColumnStats @@ -67,4 +69,25 @@ public String toString() .omitNullValues() .toString(); } + + @Override + public boolean equals(Object other) + { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + ColumnStats that = (ColumnStats) other; + return Objects.equals(columnId, that.columnId) && + Objects.equals(min, that.min) && + Objects.equals(max, that.max); + } + + @Override + public int hashCode() + { + return Objects.hash(columnId, min, max); + } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java new file mode 100644 index 0000000000000..d99219339ba9f --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.presto.spi.type.Type; + +import java.io.File; +import java.io.IOException; +import java.util.BitSet; +import java.util.Map; + +public interface FileRewriter +{ + OrcFileInfo rewrite(Map columns, File input, File output, BitSet rowsToDelete) throws IOException; +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileInfo.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileInfo.java new file mode 100644 index 0000000000000..2a96cec4023da --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileInfo.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import javax.annotation.concurrent.Immutable; + +@Immutable +public class OrcFileInfo +{ + private final long rowCount; + private final long uncompressedSize; + + public OrcFileInfo(long rowCount, long uncompressedSize) + { + this.rowCount = rowCount; + this.uncompressedSize = uncompressedSize; + } + + public long getRowCount() + { + return rowCount; + } + + public long getUncompressedSize() + { + return uncompressedSize; + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java index 739aaf2656133..c3c735b4306c5 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java @@ -13,314 +13,166 @@ */ package com.facebook.presto.raptor.storage; -import com.facebook.presto.raptor.util.SyncingFileSystem; +import com.facebook.presto.orc.OrcWriter; +import com.facebook.presto.orc.OrcWriterOptions; +import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; -import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; -import com.facebook.presto.spi.type.VarbinaryType; -import com.facebook.presto.spi.type.VarcharType; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.json.JsonCodec; -import io.airlift.slice.Slice; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.io.orc.NullMemoryManager; -import org.apache.hadoop.hive.ql.io.orc.OrcFile; -import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcSerde; -import org.apache.hadoop.hive.ql.io.orc.OrcWriterOptions; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import java.io.Closeable; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; -import java.lang.reflect.Constructor; -import java.nio.ByteBuffer; +import java.io.UncheckedIOException; import java.util.Collection; import java.util.HashSet; import java.util.List; -import java.util.Properties; +import java.util.Map; +import static com.facebook.presto.orc.OrcEncoding.ORC; +import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.HASHED; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; -import static com.facebook.presto.raptor.storage.Row.extractRow; -import static com.facebook.presto.raptor.storage.StorageType.arrayOf; -import static com.facebook.presto.raptor.storage.StorageType.mapOf; -import static com.facebook.presto.raptor.util.Types.isArrayType; -import static com.facebook.presto.raptor.util.Types.isMapType; -import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.google.common.base.Functions.toStringFunction; +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_WRITER_DATA_ERROR; +import static com.facebook.presto.raptor.storage.OrcStorageManager.DEFAULT_STORAGE_TIMEZONE; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.json.JsonCodec.jsonCodec; import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES; -import static org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; -import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.SNAPPY; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.LIST; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.MAP; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector; -import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; public class OrcFileWriter - implements Closeable + implements FileWriter { - private static final Configuration CONFIGURATION = new Configuration(); - private static final Constructor WRITER_CONSTRUCTOR = getOrcWriterConstructor(); + public static final OrcWriterOptions DEFAULT_OPTION = new OrcWriterOptions(); private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - private final List columnTypes; - - private final OrcSerde serializer; - private final RecordWriter recordWriter; - private final SettableStructObjectInspector tableInspector; - private final List structFields; - private final Object orcRow; + private final OrcWriter orcWriter; private boolean closed; private long rowCount; private long uncompressedSize; - public OrcFileWriter(List columnIds, List columnTypes, File target) + public OrcFileWriter(List columnIds, List columnTypes, File target, boolean validate, OrcWriterStats stats, TypeManager typeManager, CompressionKind compression) { - this(columnIds, columnTypes, target, true); + this(columnIds, columnTypes, target, true, validate, stats, typeManager, compression); } @VisibleForTesting - OrcFileWriter(List columnIds, List columnTypes, File target, boolean writeMetadata) - { - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - checkArgument(columnIds.size() == columnTypes.size(), "ids and types mismatch"); + OrcFileWriter( + List columnIds, + List columnTypes, + File target, + boolean writeMetadata, + boolean validate, + OrcWriterStats stats, + TypeManager typeManager, + CompressionKind compression) + { + checkArgument(requireNonNull(columnIds, "columnIds is null").size() == requireNonNull(columnTypes, "columnTypes is null").size(), "ids and types mismatch"); checkArgument(isUnique(columnIds), "ids must be unique"); - List storageTypes = ImmutableList.copyOf(toStorageTypes(columnTypes)); - Iterable hiveTypeNames = storageTypes.stream().map(StorageType::getHiveTypeName).collect(toList()); - List columnNames = ImmutableList.copyOf(transform(columnIds, toStringFunction())); - - Properties properties = new Properties(); - properties.setProperty(META_TABLE_COLUMNS, Joiner.on(',').join(columnNames)); - properties.setProperty(META_TABLE_COLUMN_TYPES, Joiner.on(':').join(hiveTypeNames)); + StorageTypeConverter converter = new StorageTypeConverter(typeManager); + List storageTypes = columnTypes.stream() + .map(converter::toStorageType) + .collect(toImmutableList()); + List columnNames = columnIds.stream().map(Object::toString).collect(toImmutableList()); - serializer = createSerializer(properties); - recordWriter = createRecordWriter(new Path(target.toURI()), columnIds, columnTypes, writeMetadata); + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { + Map userMetadata = ImmutableMap.of(); + if (writeMetadata) { + ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); + for (int i = 0; i < columnIds.size(); i++) { + columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeSignature()); + } + userMetadata = ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(new OrcFileMetadata(columnTypesMap.build()))); + } - tableInspector = getStandardStructObjectInspector(columnNames, getJavaObjectInspectors(storageTypes)); - structFields = ImmutableList.copyOf(tableInspector.getAllStructFieldRefs()); - orcRow = tableInspector.create(); + orcWriter = new OrcWriter( + new OutputStreamOrcDataSink(new FileOutputStream(target)), + columnNames, + storageTypes, + ORC, + requireNonNull(compression, "compression is null"), + DEFAULT_OPTION, + userMetadata, + DEFAULT_STORAGE_TIMEZONE, + validate, + HASHED, + stats); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); + } } + @Override public void appendPages(List pages) { for (Page page : pages) { - for (int position = 0; position < page.getPositionCount(); position++) { - appendRow(extractRow(page, position, columnTypes)); + try { + orcWriter.write(page); } + catch (IOException | UncheckedIOException e) { + throw new PrestoException(RAPTOR_WRITER_DATA_ERROR, e); + } + uncompressedSize += page.getLogicalSizeInBytes(); + rowCount += page.getPositionCount(); } } + @Override public void appendPages(List inputPages, int[] pageIndexes, int[] positionIndexes) { checkArgument(pageIndexes.length == positionIndexes.length, "pageIndexes and positionIndexes do not match"); for (int i = 0; i < pageIndexes.length; i++) { Page page = inputPages.get(pageIndexes[i]); - appendRow(extractRow(page, positionIndexes[i], columnTypes)); - } - } - - public void appendRow(Row row) - { - List columns = row.getColumns(); - checkArgument(columns.size() == columnTypes.size()); - for (int channel = 0; channel < columns.size(); channel++) { - tableInspector.setStructFieldData(orcRow, structFields.get(channel), columns.get(channel)); - } - try { - recordWriter.write(serializer.serialize(orcRow, tableInspector)); - } - catch (IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to write record", e); + // This will do data copy; be aware + Page singleValuePage = page.getSingleValuePage(positionIndexes[i]); + try { + orcWriter.write(singleValuePage); + uncompressedSize += singleValuePage.getLogicalSizeInBytes(); + rowCount++; + } + catch (IOException | UncheckedIOException e) { + throw new PrestoException(RAPTOR_WRITER_DATA_ERROR, e); + } } - rowCount++; - uncompressedSize += row.getSizeInBytes(); } @Override public void close() + throws IOException { if (closed) { return; } closed = true; - try { - recordWriter.close(false); - } - catch (IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to close writer", e); - } + orcWriter.close(); } + @Override public long getRowCount() { return rowCount; } + @Override public long getUncompressedSize() { return uncompressedSize; } - private static OrcSerde createSerializer(Properties properties) - { - OrcSerde serde = new OrcSerde(); - serde.initialize(CONFIGURATION, properties); - return serde; - } - - private static RecordWriter createRecordWriter(Path target, List columnIds, List columnTypes, boolean writeMetadata) - { - try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); - FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) { - OrcFile.WriterOptions options = new OrcWriterOptions(CONFIGURATION) - .memory(new NullMemoryManager(CONFIGURATION)) - .fileSystem(fileSystem) - .compress(SNAPPY); - - if (writeMetadata) { - options.callback(createFileMetadataCallback(columnIds, columnTypes)); - } - - return WRITER_CONSTRUCTOR.newInstance(target, options); - } - catch (ReflectiveOperationException | IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); - } - } - - private static OrcFile.WriterCallback createFileMetadataCallback(List columnIds, List columnTypes) - { - return new OrcFile.WriterCallback() - { - @Override - public void preStripeWrite(OrcFile.WriterContext context) - {} - - @Override - public void preFooterWrite(OrcFile.WriterContext context) - { - ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); - for (int i = 0; i < columnIds.size(); i++) { - columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeSignature()); - } - byte[] bytes = METADATA_CODEC.toJsonBytes(new OrcFileMetadata(columnTypesMap.build())); - context.getWriter().addUserMetadata(OrcFileMetadata.KEY, ByteBuffer.wrap(bytes)); - } - }; - } - - private static Constructor getOrcWriterConstructor() - { - try { - String writerClassName = OrcOutputFormat.class.getName() + "$OrcRecordWriter"; - Constructor constructor = OrcOutputFormat.class.getClassLoader() - .loadClass(writerClassName).asSubclass(RecordWriter.class) - .getDeclaredConstructor(Path.class, OrcFile.WriterOptions.class); - constructor.setAccessible(true); - return constructor; - } - catch (ReflectiveOperationException e) { - throw new RuntimeException(e); - } - } - - private static List getJavaObjectInspectors(List types) - { - return types.stream() - .map(StorageType::getHiveTypeName) - .map(TypeInfoUtils::getTypeInfoFromTypeString) - .map(OrcFileWriter::getJavaObjectInspector) - .collect(toList()); - } - - private static ObjectInspector getJavaObjectInspector(TypeInfo typeInfo) - { - Category category = typeInfo.getCategory(); - if (category == PRIMITIVE) { - return getPrimitiveJavaObjectInspector(getPrimitiveTypeInfo(typeInfo.getTypeName())); - } - if (category == LIST) { - ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; - return getStandardListObjectInspector(getJavaObjectInspector(listTypeInfo.getListElementTypeInfo())); - } - if (category == MAP) { - MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; - return getStandardMapObjectInspector( - getJavaObjectInspector(mapTypeInfo.getMapKeyTypeInfo()), - getJavaObjectInspector(mapTypeInfo.getMapValueTypeInfo())); - } - throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unhandled storage type: " + category); - } - private static boolean isUnique(Collection items) { return new HashSet<>(items).size() == items.size(); } - - private static List toStorageTypes(List columnTypes) - { - return columnTypes.stream().map(OrcFileWriter::toStorageType).collect(toList()); - } - - private static StorageType toStorageType(Type type) - { - if (type instanceof DecimalType) { - DecimalType decimalType = (DecimalType) type; - return StorageType.decimal(decimalType.getPrecision(), decimalType.getScale()); - } - Class javaType = type.getJavaType(); - if (javaType == boolean.class) { - return StorageType.BOOLEAN; - } - if (javaType == long.class) { - return StorageType.LONG; - } - if (javaType == double.class) { - return StorageType.DOUBLE; - } - if (javaType == Slice.class) { - if (type instanceof VarcharType) { - return StorageType.STRING; - } - if (type.equals(VarbinaryType.VARBINARY)) { - return StorageType.BYTES; - } - } - if (isArrayType(type)) { - return arrayOf(toStorageType(type.getTypeParameters().get(0))); - } - if (isMapType(type)) { - return mapOf(toStorageType(type.getTypeParameters().get(0)), toStorageType(type.getTypeParameters().get(1))); - } - throw new PrestoException(NOT_SUPPORTED, "No storage type for type: " + type); - } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java new file mode 100644 index 0000000000000..16dab6d41bbc7 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java @@ -0,0 +1,239 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.presto.orc.FileOrcDataSource; +import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.orc.OrcWriter; +import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.orc.metadata.CompressionKind; +import com.facebook.presto.raptor.util.Closer; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignature; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.json.JsonCodec; +import io.airlift.log.Logger; +import org.apache.hadoop.fs.FileSystem; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.BitSet; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.IntStream; + +import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.orc.OrcEncoding.ORC; +import static com.facebook.presto.orc.OrcPredicate.TRUE; +import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; +import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.HASHED; +import static com.facebook.presto.raptor.storage.OrcFileWriter.DEFAULT_OPTION; +import static com.facebook.presto.raptor.storage.OrcStorageManager.DEFAULT_STORAGE_TIMEZONE; +import static com.facebook.presto.raptor.storage.OrcStorageManager.HUGE_MAX_READ_BLOCK_SIZE; +import static com.facebook.presto.raptor.util.Closer.closer; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.json.JsonCodec.jsonCodec; +import static io.airlift.units.Duration.nanosSince; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; + +public final class OrcPageFileRewriter + implements FileRewriter +{ + private static final Logger log = Logger.get(OrcPageFileRewriter.class); + private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); + + private final ReaderAttributes readerAttributes; + private final boolean validate; + private final OrcWriterStats stats; + private final TypeManager typeManager; + private final CompressionKind compression; + + OrcPageFileRewriter( + ReaderAttributes readerAttributes, + boolean validate, + OrcWriterStats stats, + TypeManager typeManager, + CompressionKind compression) + { + this.readerAttributes = requireNonNull(readerAttributes, "readerAttributes is null"); + this.validate = validate; + this.stats = requireNonNull(stats, "stats is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.compression = requireNonNull(compression, "compression is null"); + } + + @Override + public OrcFileInfo rewrite(Map allColumnTypes, File input, File output, BitSet rowsToDelete) + throws IOException + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { + OrcReader reader = new OrcReader( + new FileOrcDataSource(input, readerAttributes.getMaxMergeDistance(), readerAttributes.getMaxReadSize(), readerAttributes.getStreamBufferSize(), readerAttributes.isLazyReadSmallRanges()), + ORC, + readerAttributes.getMaxMergeDistance(), + readerAttributes.getMaxReadSize(), + readerAttributes.getTinyStripeThreshold(), + HUGE_MAX_READ_BLOCK_SIZE); + + if (reader.getFooter().getNumberOfRows() < rowsToDelete.length()) { + throw new IOException("File has fewer rows than deletion vector"); + } + int deleteRowCount = rowsToDelete.cardinality(); + if (reader.getFooter().getNumberOfRows() == deleteRowCount) { + return new OrcFileInfo(0, 0); + } + if (reader.getFooter().getNumberOfRows() >= Integer.MAX_VALUE) { + throw new IOException("File has too many rows"); + } + int inputRowCount = toIntExact(reader.getFooter().getNumberOfRows()); + + Map currentColumnIds = IntStream.range(0, reader.getColumnNames().size()).boxed().collect(toMap(reader.getColumnNames()::get, i -> i)); + + ImmutableList.Builder writerColumnTypesBuilder = ImmutableList.builder(); + ImmutableList.Builder writerColumnIdsBuilder = ImmutableList.builder(); + ImmutableList.Builder readerColumnIndexBuilder = ImmutableList.builder(); + + // Build columns for writer; keep the right ordinal + Map orderedAllColumnTypes = new TreeMap<>(Comparator.comparingLong(Long::parseLong)); + orderedAllColumnTypes.putAll(allColumnTypes); + for (Map.Entry columnType : orderedAllColumnTypes.entrySet()) { + // Get the intersection of the provide columns and the actual columns + Integer currentColumnIndex = currentColumnIds.get(columnType.getKey()); + if (currentColumnIndex != null) { + readerColumnIndexBuilder.add(currentColumnIndex); + writerColumnTypesBuilder.add(columnType.getValue()); + writerColumnIdsBuilder.add(columnType.getKey()); + } + } + + List writerColumnTypes = writerColumnTypesBuilder.build(); + List writerColumnIds = writerColumnIdsBuilder.build(); + List readerColumnIndex = readerColumnIndexBuilder.build(); + Map readerColumns = IntStream.range(0, readerColumnIndex.size()).boxed().collect(toMap(readerColumnIndex::get, writerColumnTypes::get)); + + if (writerColumnTypes.isEmpty()) { + // no intersection; directly return + return new OrcFileInfo(0, 0); + } + + StorageTypeConverter converter = new StorageTypeConverter(typeManager); + List writerStorageTypes = writerColumnTypes.stream() + .map(converter::toStorageType) + .collect(toImmutableList()); + + long start = System.nanoTime(); + + Map userMetadata = ImmutableMap.of(); + if (reader.getFooter().getUserMetadata().containsKey(OrcFileMetadata.KEY)) { + // build metadata if the original file has it + ImmutableMap.Builder metadataBuilder = ImmutableMap.builder(); + for (int i = 0; i < writerColumnIds.size(); i++) { + metadataBuilder.put(Long.parseLong(writerColumnIds.get(i)), writerColumnTypes.get(i).getTypeSignature()); + } + userMetadata = ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(new OrcFileMetadata(metadataBuilder.build()))); + } + try (Closer recordReader = closer(reader.createRecordReader(readerColumns, TRUE, DEFAULT_STORAGE_TIMEZONE, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE), OrcRecordReader::close); + Closer writer = closer(new OrcWriter( + new OutputStreamOrcDataSink(new FileOutputStream(output)), + writerColumnIds, + writerStorageTypes, + ORC, + compression, + DEFAULT_OPTION, + userMetadata, + DEFAULT_STORAGE_TIMEZONE, + validate, + HASHED, + stats), + OrcWriter::close)) { + OrcFileInfo fileInfo = rewrite(recordReader.get(), writer.get(), rowsToDelete, writerColumnTypes, readerColumnIndexBuilder.build()); + log.debug("Rewrote file %s in %s (input rows: %s, output rows: %s)", input.getName(), nanosSince(start), inputRowCount, inputRowCount - deleteRowCount); + return fileInfo; + } + } + } + + private static OrcFileInfo rewrite( + OrcRecordReader reader, + OrcWriter writer, + BitSet rowsToDelete, + List types, + List readerColumnIndex) + throws IOException + { + int row = rowsToDelete.nextClearBit(0); + long rowCount = 0; + long uncompressedSize = 0; + + while (true) { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedIOException(); + } + + int batchSize = reader.nextBatch(); + if (batchSize <= 0) { + break; + } + + if (row >= (reader.getFilePosition() + batchSize)) { + continue; + } + + Block[] blocks = new Block[types.size()]; + for (int i = 0; i < types.size(); i++) { + // read from existing columns + blocks[i] = reader.readBlock(types.get(i), readerColumnIndex.get(i)); + } + + row = toIntExact(reader.getFilePosition()); + + Page page = maskedPage(blocks, rowsToDelete, row, batchSize); + writer.write(page); + + rowCount += page.getPositionCount(); + uncompressedSize += page.getLogicalSizeInBytes(); + + row = rowsToDelete.nextClearBit(row + batchSize); + } + + return new OrcFileInfo(rowCount, uncompressedSize); + } + + private static Page maskedPage(Block[] blocks, BitSet rowsToDelete, int start, int count) + { + int[] ids = new int[count]; + int size = 0; + for (int i = 0; i < count; i++) { + if (!rowsToDelete.get(start + i)) { + ids[size] = i; + size++; + } + } + + return new Page(blocks).getPositions(ids, 0, size); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java similarity index 84% rename from presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java rename to presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java index 7ac52c6b57863..cb0c62cb99ec3 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java @@ -16,6 +16,7 @@ import com.facebook.presto.raptor.util.Closer; import com.facebook.presto.raptor.util.SyncingFileSystem; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.type.Type; import io.airlift.log.Logger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -27,10 +28,16 @@ import org.apache.hadoop.hive.ql.io.orc.Reader; import org.apache.hadoop.hive.ql.io.orc.RecordReader; import org.apache.hadoop.hive.ql.io.orc.Writer; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.ByteWritable; import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; @@ -45,21 +52,24 @@ import static com.facebook.presto.raptor.util.Closer.closer; import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; import static io.airlift.slice.SizeOf.SIZE_OF_DOUBLE; +import static io.airlift.slice.SizeOf.SIZE_OF_FLOAT; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; import static io.airlift.slice.SizeOf.SIZE_OF_LONG; +import static io.airlift.slice.SizeOf.SIZE_OF_SHORT; import static io.airlift.units.Duration.nanosSince; import static java.lang.Math.toIntExact; import static org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader; import static org.apache.hadoop.hive.ql.io.orc.OrcFile.createWriter; import static org.apache.hadoop.hive.ql.io.orc.OrcUtil.getFieldValue; -public final class OrcFileRewriter +public final class OrcRecordFileRewriter + implements FileRewriter { - private static final Logger log = Logger.get(OrcFileRewriter.class); + private static final Logger log = Logger.get(OrcRecordFileRewriter.class); private static final Configuration CONFIGURATION = new Configuration(); - private OrcFileRewriter() {} - - public static OrcFileInfo rewrite(File input, File output, BitSet rowsToDelete) + @Override + public OrcFileInfo rewrite(Map columns, File input, File output, BitSet rowsToDelete) throws IOException { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); @@ -166,6 +176,24 @@ private static int uncompressedSize(Object object) if (object instanceof BytesWritable) { return ((BytesWritable) object).getLength(); } + if (object instanceof ByteWritable) { + return SIZE_OF_BYTE; + } + if (object instanceof HiveVarcharWritable) { + return ((HiveVarcharWritable) object).getTextValue().getLength(); + } + if (object instanceof DateWritable) { + return SIZE_OF_INT; + } + if (object instanceof IntWritable) { + return SIZE_OF_INT; + } + if (object instanceof ShortWritable) { + return SIZE_OF_SHORT; + } + if (object instanceof FloatWritable) { + return SIZE_OF_FLOAT; + } if (object instanceof List) { int size = 0; for (Object element : (Iterable) object) { @@ -183,26 +211,4 @@ private static int uncompressedSize(Object object) } throw new IOException("Unhandled ORC object: " + object.getClass().getName()); } - - public static class OrcFileInfo - { - private final long rowCount; - private final long uncompressedSize; - - public OrcFileInfo(long rowCount, long uncompressedSize) - { - this.rowCount = rowCount; - this.uncompressedSize = uncompressedSize; - } - - public long getRowCount() - { - return rowCount; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java new file mode 100644 index 0000000000000..ba565c72d8673 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java @@ -0,0 +1,347 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.presto.orc.metadata.CompressionKind; +import com.facebook.presto.raptor.util.SyncingFileSystem; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.TimestampWithTimeZoneType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.spi.type.VarbinaryType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.json.JsonCodec; +import io.airlift.slice.Slice; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.orc.NullMemoryManager; +import org.apache.hadoop.hive.ql.io.orc.OrcFile; +import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; +import org.apache.hadoop.hive.ql.io.orc.OrcWriterOptions; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; + +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_UNSUPPORTED_COMPRESSION_KIND; +import static com.facebook.presto.raptor.storage.Row.extractRow; +import static com.facebook.presto.raptor.storage.StorageType.arrayOf; +import static com.facebook.presto.raptor.storage.StorageType.mapOf; +import static com.facebook.presto.raptor.util.Types.isArrayType; +import static com.facebook.presto.raptor.util.Types.isMapType; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.google.common.base.Functions.toStringFunction; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.Iterables.transform; +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES; +import static org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; +import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.NONE; +import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.SNAPPY; +import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.ZLIB; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.LIST; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.MAP; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; + +public class OrcRecordWriter + implements FileWriter +{ + private static final Configuration CONFIGURATION = new Configuration(); + private static final Constructor WRITER_CONSTRUCTOR = getOrcWriterConstructor(); + private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); + + private final List columnTypes; + + private final OrcSerde serializer; + private final RecordWriter recordWriter; + private final SettableStructObjectInspector tableInspector; + private final List structFields; + private final Object orcRow; + + private boolean closed; + private long rowCount; + private long uncompressedSize; + + public OrcRecordWriter(List columnIds, List columnTypes, File target, CompressionKind compression) + { + this(columnIds, columnTypes, target, compression, true); + } + + @VisibleForTesting + OrcRecordWriter(List columnIds, List columnTypes, File target, CompressionKind compression, boolean writeMetadata) + { + this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); + checkArgument(columnIds.size() == columnTypes.size(), "ids and types mismatch"); + checkArgument(isUnique(columnIds), "ids must be unique"); + + List storageTypes = ImmutableList.copyOf(toStorageTypes(columnTypes)); + Iterable hiveTypeNames = storageTypes.stream().map(StorageType::getHiveTypeName).collect(toList()); + List columnNames = ImmutableList.copyOf(transform(columnIds, toStringFunction())); + + Properties properties = new Properties(); + properties.setProperty(META_TABLE_COLUMNS, Joiner.on(',').join(columnNames)); + properties.setProperty(META_TABLE_COLUMN_TYPES, Joiner.on(':').join(hiveTypeNames)); + + serializer = createSerializer(properties); + recordWriter = createRecordWriter(new Path(target.toURI()), columnIds, columnTypes, requireNonNull(compression, "compression is null"), writeMetadata); + + tableInspector = getStandardStructObjectInspector(columnNames, getJavaObjectInspectors(storageTypes)); + structFields = ImmutableList.copyOf(tableInspector.getAllStructFieldRefs()); + orcRow = tableInspector.create(); + } + + @Override + public void appendPages(List pages) + { + for (Page page : pages) { + for (int position = 0; position < page.getPositionCount(); position++) { + appendRow(extractRow(page, position, columnTypes)); + } + } + } + + @Override + public void appendPages(List inputPages, int[] pageIndexes, int[] positionIndexes) + { + checkArgument(pageIndexes.length == positionIndexes.length, "pageIndexes and positionIndexes do not match"); + for (int i = 0; i < pageIndexes.length; i++) { + Page page = inputPages.get(pageIndexes[i]); + appendRow(extractRow(page, positionIndexes[i], columnTypes)); + } + } + + @Override + public void close() + throws IOException + { + if (closed) { + return; + } + closed = true; + + recordWriter.close(false); + } + + @Override + public long getRowCount() + { + return rowCount; + } + + @Override + public long getUncompressedSize() + { + return uncompressedSize; + } + + private void appendRow(Row row) + { + List columns = row.getColumns(); + checkArgument(columns.size() == columnTypes.size()); + for (int channel = 0; channel < columns.size(); channel++) { + tableInspector.setStructFieldData(orcRow, structFields.get(channel), columns.get(channel)); + } + try { + recordWriter.write(serializer.serialize(orcRow, tableInspector)); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to write record", e); + } + rowCount++; + uncompressedSize += row.getSizeInBytes(); + } + + private static OrcSerde createSerializer(Properties properties) + { + OrcSerde serde = new OrcSerde(); + serde.initialize(CONFIGURATION, properties); + return serde; + } + + private static RecordWriter createRecordWriter(Path target, List columnIds, List columnTypes, CompressionKind compression, boolean writeMetadata) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); + FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) { + OrcFile.WriterOptions options = new OrcWriterOptions(CONFIGURATION) + .memory(new NullMemoryManager(CONFIGURATION)) + .fileSystem(fileSystem) + .compress(toCompressionKind(compression)); + + if (writeMetadata) { + options.callback(createFileMetadataCallback(columnIds, columnTypes)); + } + + return WRITER_CONSTRUCTOR.newInstance(target, options); + } + catch (ReflectiveOperationException | IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); + } + } + + private static org.apache.hadoop.hive.ql.io.orc.CompressionKind toCompressionKind(CompressionKind compression) + { + switch (compression) { + case NONE: + return NONE; + case SNAPPY: + return SNAPPY; + case ZLIB: + return ZLIB; + default: + throw new PrestoException(RAPTOR_UNSUPPORTED_COMPRESSION_KIND, "Found unsupported compression kind: " + compression); + } + } + + private static OrcFile.WriterCallback createFileMetadataCallback(List columnIds, List columnTypes) + { + return new OrcFile.WriterCallback() + { + @Override + public void preStripeWrite(OrcFile.WriterContext context) + {} + + @Override + public void preFooterWrite(OrcFile.WriterContext context) + { + ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); + for (int i = 0; i < columnIds.size(); i++) { + columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeSignature()); + } + byte[] bytes = METADATA_CODEC.toJsonBytes(new OrcFileMetadata(columnTypesMap.build())); + context.getWriter().addUserMetadata(OrcFileMetadata.KEY, ByteBuffer.wrap(bytes)); + } + }; + } + + private static Constructor getOrcWriterConstructor() + { + try { + String writerClassName = OrcOutputFormat.class.getName() + "$OrcRecordWriter"; + Constructor constructor = OrcOutputFormat.class.getClassLoader() + .loadClass(writerClassName).asSubclass(RecordWriter.class) + .getDeclaredConstructor(Path.class, OrcFile.WriterOptions.class); + constructor.setAccessible(true); + return constructor; + } + catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + + private static List getJavaObjectInspectors(List types) + { + return types.stream() + .map(StorageType::getHiveTypeName) + .map(TypeInfoUtils::getTypeInfoFromTypeString) + .map(OrcRecordWriter::getJavaObjectInspector) + .collect(toList()); + } + + private static ObjectInspector getJavaObjectInspector(TypeInfo typeInfo) + { + Category category = typeInfo.getCategory(); + if (category == PRIMITIVE) { + return getPrimitiveJavaObjectInspector(getPrimitiveTypeInfo(typeInfo.getTypeName())); + } + if (category == LIST) { + ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; + return getStandardListObjectInspector(getJavaObjectInspector(listTypeInfo.getListElementTypeInfo())); + } + if (category == MAP) { + MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; + return getStandardMapObjectInspector( + getJavaObjectInspector(mapTypeInfo.getMapKeyTypeInfo()), + getJavaObjectInspector(mapTypeInfo.getMapValueTypeInfo())); + } + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unhandled storage type: " + category); + } + + private static boolean isUnique(Collection items) + { + return new HashSet<>(items).size() == items.size(); + } + + private static List toStorageTypes(List columnTypes) + { + return columnTypes.stream().map(OrcRecordWriter::toStorageType).collect(toList()); + } + + private static StorageType toStorageType(Type type) + { + if (type instanceof DecimalType) { + DecimalType decimalType = (DecimalType) type; + return StorageType.decimal(decimalType.getPrecision(), decimalType.getScale()); + } + if (type instanceof TimestampWithTimeZoneType) { + throw new PrestoException(NOT_SUPPORTED, "The current ORC writer does not support type: " + type); + } + Class javaType = type.getJavaType(); + if (javaType == boolean.class) { + return StorageType.BOOLEAN; + } + if (javaType == long.class) { + return StorageType.LONG; + } + if (javaType == double.class) { + return StorageType.DOUBLE; + } + if (javaType == Slice.class) { + if (type instanceof VarcharType) { + return StorageType.STRING; + } + if (type.equals(VarbinaryType.VARBINARY)) { + return StorageType.BYTES; + } + } + if (isArrayType(type)) { + return arrayOf(toStorageType(type.getTypeParameters().get(0))); + } + if (isMapType(type)) { + return mapOf(toStorageType(type.getTypeParameters().get(0)), toStorageType(type.getTypeParameters().get(1))); + } + throw new PrestoException(NOT_SUPPORTED, "No storage type for type: " + type); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index 91f6566aebe12..866324caabae1 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -19,8 +19,10 @@ import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.orc.TupleDomainOrcPredicate; import com.facebook.presto.orc.TupleDomainOrcPredicate.ColumnReference; +import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.orc.metadata.OrcType; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.RaptorConnectorId; @@ -31,7 +33,7 @@ import com.facebook.presto.raptor.metadata.ShardDelta; import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.metadata.ShardRecorder; -import com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo; +import com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.Page; @@ -55,6 +57,7 @@ import io.airlift.slice.XxHash64; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import org.joda.time.DateTimeZone; import javax.annotation.PreDestroy; import javax.inject.Inject; @@ -99,6 +102,8 @@ import static com.facebook.presto.raptor.storage.OrcPageSource.ROWID_COLUMN; import static com.facebook.presto.raptor.storage.OrcPageSource.SHARD_UUID_COLUMN; import static com.facebook.presto.raptor.storage.ShardStats.computeColumnStats; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.CharType.createCharType; @@ -127,11 +132,17 @@ public class OrcStorageManager implements StorageManager { + // Raptor does not store time-related data types as they are in ORC. + // They will be casted to BIGINT or INTEGER to avoid timezone conversion. + // This is due to the historical reason of using the legacy ORC read/writer that does not support timestamp types. + // In order to be consistent, we still enforce the conversion. + // The following DEFAULT_STORAGE_TIMEZONE is not used by the optimized ORC read/writer given we never read/write timestamp types. + public static final DateTimeZone DEFAULT_STORAGE_TIMEZONE = UTC; + // TODO: do not limit the max size of blocks to read for now; enable the limit when the Hive connector is ready + public static final DataSize HUGE_MAX_READ_BLOCK_SIZE = new DataSize(1, PETABYTE); private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); private static final long MAX_ROWS = 1_000_000_000; - // TODO: do not limit the max size of blocks to read for now; enable the limit when the Hive connector is ready - private static final DataSize HUGE_MAX_READ_BLOCK_SIZE = new DataSize(1, PETABYTE); private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); private final String nodeId; @@ -145,9 +156,13 @@ public class OrcStorageManager private final long maxShardRows; private final DataSize maxShardSize; private final DataSize minAvailableSpace; + private final CompressionKind compression; + private final OrcOptimizedWriterStage orcOptimizedWriterStage; private final TypeManager typeManager; private final ExecutorService deletionExecutor; private final ExecutorService commitExecutor; + private final FileRewriter fileRewriter; + private final OrcWriterStats stats = new OrcWriterStats(); @Inject public OrcStorageManager( @@ -175,7 +190,9 @@ public OrcStorageManager( config.getShardRecoveryTimeout(), config.getMaxShardRows(), config.getMaxShardSize(), - config.getMinAvailableSpace()); + config.getMinAvailableSpace(), + config.getOrcCompressionKind(), + config.getOrcOptimizedWriterStage()); } public OrcStorageManager( @@ -192,7 +209,9 @@ public OrcStorageManager( Duration shardRecoveryTimeout, long maxShardRows, DataSize maxShardSize, - DataSize minAvailableSpace) + DataSize minAvailableSpace, + CompressionKind compression, + OrcOptimizedWriterStage orcOptimizedWriterStage) { this.nodeId = requireNonNull(nodeId, "nodeId is null"); this.storageService = requireNonNull(storageService, "storageService is null"); @@ -211,6 +230,14 @@ public OrcStorageManager( this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.deletionExecutor = newFixedThreadPool(deletionThreads, daemonThreadsNamed("raptor-delete-" + connectorId + "-%s")); this.commitExecutor = newCachedThreadPool(daemonThreadsNamed("raptor-commit-" + connectorId + "-%s")); + this.compression = requireNonNull(compression, "compression is null"); + this.orcOptimizedWriterStage = requireNonNull(orcOptimizedWriterStage, "orcOptimizedWriterStage is null"); + if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { + this.fileRewriter = new OrcPageFileRewriter(readerAttributes, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager, compression); + } + else { + this.fileRewriter = new OrcRecordFileRewriter(); + } } @PreDestroy @@ -228,7 +255,8 @@ public ConnectorPageSource getPageSource( List columnTypes, TupleDomain effectivePredicate, ReaderAttributes readerAttributes, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> allColumnTypes) { OrcDataSource dataSource = openShard(shardUuid, readerAttributes); @@ -259,11 +287,12 @@ public ConnectorPageSource getPageSource( OrcPredicate predicate = getPredicate(effectivePredicate, indexMap); - OrcRecordReader recordReader = reader.createRecordReader(includedColumns.build(), predicate, UTC, systemMemoryUsage, INITIAL_BATCH_SIZE); + OrcRecordReader recordReader = reader.createRecordReader(includedColumns.build(), predicate, DEFAULT_STORAGE_TIMEZONE, systemMemoryUsage, INITIAL_BATCH_SIZE); Optional shardRewriter = Optional.empty(); if (transactionId.isPresent()) { - shardRewriter = Optional.of(createShardRewriter(transactionId.getAsLong(), bucketNumber, shardUuid)); + checkState(allColumnTypes.isPresent()); + shardRewriter = Optional.of(createShardRewriter(transactionId.getAsLong(), bucketNumber, shardUuid, allColumnTypes.get())); } return new OrcPageSource(shardRewriter, recordReader, dataSource, columnIds, columnTypes, columnIndexes.build(), shardUuid, bucketNumber, systemMemoryUsage); @@ -301,13 +330,13 @@ public StoragePageSink createStoragePageSink(long transactionId, OptionalInt buc return new OrcStoragePageSink(transactionId, columnIds, columnTypes, bucketNumber); } - private ShardRewriter createShardRewriter(long transactionId, OptionalInt bucketNumber, UUID shardUuid) + private ShardRewriter createShardRewriter(long transactionId, OptionalInt bucketNumber, UUID shardUuid, Map columns) { return rowsToDelete -> { if (rowsToDelete.isEmpty()) { return completedFuture(ImmutableList.of()); } - return supplyAsync(() -> rewriteShard(transactionId, bucketNumber, shardUuid, rowsToDelete), deletionExecutor); + return supplyAsync(() -> rewriteShard(transactionId, bucketNumber, shardUuid, columns, rowsToDelete), deletionExecutor); }; } @@ -391,7 +420,7 @@ private List computeShardStats(File file) } @VisibleForTesting - Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, BitSet rowsToDelete) + Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, Map columns, BitSet rowsToDelete) { if (rowsToDelete.isEmpty()) { return ImmutableList.of(); @@ -401,7 +430,7 @@ Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUI File input = storageService.getStorageFile(shardUuid); File output = storageService.getStagingFile(newShardUuid); - OrcFileInfo info = rewriteFile(input, output, rowsToDelete); + OrcFileInfo info = rewriteFile(columns, input, output, rowsToDelete); long rowCount = info.getRowCount(); if (rowCount == 0) { @@ -430,10 +459,10 @@ private static Collection shardDelta(UUID oldShardUuid, Optional columns, File input, File output, BitSet rowsToDelete) { try { - return OrcFileRewriter.rewrite(input, output, rowsToDelete); + return fileRewriter.rewrite(columns, input, output, rowsToDelete); } catch (IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to rewrite shard file: " + input, e); @@ -564,10 +593,14 @@ private class OrcStoragePageSink private final List> futures = new ArrayList<>(); private boolean committed; - private OrcFileWriter writer; + private FileWriter writer; private UUID shardUuid; - public OrcStoragePageSink(long transactionId, List columnIds, List columnTypes, OptionalInt bucketNumber) + public OrcStoragePageSink( + long transactionId, + List columnIds, + List columnTypes, + OptionalInt bucketNumber) { this.transactionId = transactionId; this.columnIds = ImmutableList.copyOf(requireNonNull(columnIds, "columnIds is null")); @@ -589,13 +622,6 @@ public void appendPages(List inputPages, int[] pageIndexes, int[] position writer.appendPages(inputPages, pageIndexes, positionIndexes); } - @Override - public void appendRow(Row row) - { - createWriterIfNecessary(); - writer.appendRow(row); - } - @Override public boolean isFull() { @@ -609,7 +635,12 @@ public boolean isFull() public void flush() { if (writer != null) { - writer.close(); + try { + writer.close(); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to close writer", e); + } shardRecorder.recordCreatedShard(transactionId, shardUuid); @@ -649,8 +680,15 @@ public void rollback() { try { if (writer != null) { - writer.close(); - writer = null; + try { + writer.close(); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to close writer", e); + } + finally { + writer = null; + } } } finally { @@ -677,7 +715,12 @@ private void createWriterIfNecessary() File stagingFile = storageService.getStagingFile(shardUuid); storageService.createParents(stagingFile); stagingFiles.add(stagingFile); - writer = new OrcFileWriter(columnIds, columnTypes, stagingFile); + if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { + writer = new OrcFileWriter(columnIds, columnTypes, stagingFile, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager, compression); + } + else { + writer = new OrcRecordWriter(columnIds, columnTypes, stagingFile, compression); + } } } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java index aeaee6ea3e800..0de7002784204 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.spi.type.DateType; import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.TimeType; import com.facebook.presto.spi.type.TimestampType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; @@ -74,6 +75,7 @@ private static ColumnStats doComputeColumnStats(OrcReader orcReader, long column } if (type.equals(BigintType.BIGINT) || type.equals(DateType.DATE) || + type.equals(TimeType.TIME) || type.equals(TimestampType.TIMESTAMP)) { return indexLong(type, reader, columnIndex, columnId); } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java index 9c938f4dc9713..47e5bd981e3be 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java @@ -19,6 +19,8 @@ import com.facebook.presto.spi.type.Type; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.UUID; @@ -33,7 +35,7 @@ default ConnectorPageSource getPageSource( TupleDomain effectivePredicate, ReaderAttributes readerAttributes) { - return getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, effectivePredicate, readerAttributes, OptionalLong.empty()); + return getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, effectivePredicate, readerAttributes, OptionalLong.empty(), Optional.empty()); } ConnectorPageSource getPageSource( @@ -43,7 +45,8 @@ ConnectorPageSource getPageSource( List columnTypes, TupleDomain effectivePredicate, ReaderAttributes readerAttributes, - OptionalLong transactionId); + OptionalLong transactionId, + Optional> allColumnTypes); StoragePageSink createStoragePageSink( long transactionId, diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java index 70d930c811496..fe3d773240737 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.spi.type.TimeZoneKey; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; @@ -33,6 +34,8 @@ import java.util.TimeZone; import java.util.concurrent.TimeUnit; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.Math.max; @@ -53,6 +56,8 @@ public class StorageManagerConfig private DataSize orcStreamBufferSize = new DataSize(8, MEGABYTE); private DataSize orcTinyStripeThreshold = new DataSize(8, MEGABYTE); private boolean orcLazyReadSmallRanges = true; + private OrcOptimizedWriterStage orcOptimizedWriterStage = DISABLED; + private CompressionKind orcCompressionKind = SNAPPY; private int deletionThreads = max(1, getRuntime().availableProcessors() / 2); private int recoveryThreads = 10; private int organizationThreads = 5; @@ -161,6 +166,35 @@ public StorageManagerConfig setOrcLazyReadSmallRanges(boolean orcLazyReadSmallRa return this; } + public enum OrcOptimizedWriterStage + { + DISABLED, ENABLED, ENABLED_AND_VALIDATED + } + + public OrcOptimizedWriterStage getOrcOptimizedWriterStage() + { + return orcOptimizedWriterStage; + } + + @Config("storage.orc.optimized-writer-stage") + public StorageManagerConfig setOrcOptimizedWriterStage(OrcOptimizedWriterStage orcOptimizedWriterStage) + { + this.orcOptimizedWriterStage = orcOptimizedWriterStage; + return this; + } + + public CompressionKind getOrcCompressionKind() + { + return orcCompressionKind; + } + + @Config("storage.orc.compression-kind") + public StorageManagerConfig setOrcCompressionKind(CompressionKind orcCompressionKind) + { + this.orcCompressionKind = orcCompressionKind; + return this; + } + @Min(1) public int getDeletionThreads() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StoragePageSink.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StoragePageSink.java index 88232ff0f135f..4787e8ffced4d 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StoragePageSink.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StoragePageSink.java @@ -25,8 +25,6 @@ public interface StoragePageSink void appendPages(List pages, int[] pageIndexes, int[] positionIndexes); - void appendRow(Row row); - boolean isFull(); void flush(); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java new file mode 100644 index 0000000000000..71ff53c91088f --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignatureParameter; +import com.google.common.collect.ImmutableList; + +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.Chars.isCharType; +import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.Decimals.isLongDecimal; +import static com.facebook.presto.spi.type.Decimals.isShortDecimal; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TimeType.TIME; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class StorageTypeConverter +{ + private final TypeManager typeManager; + + public StorageTypeConverter(TypeManager typeManager) + { + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + public Type toStorageType(Type type) + { + Type storageType; + if (type.equals(BOOLEAN) || + type.equals(BIGINT) || + type.equals(INTEGER) || + type.equals(SMALLINT) || + type.equals(TINYINT) || + type.equals(DOUBLE) || + type.equals(REAL) || + type.equals(DATE) || + isCharType(type) || + isVarcharType(type) || + isLongDecimal(type) || + isShortDecimal(type) || + type.equals(VARBINARY)) { + storageType = type; + } + // Raptor does not handle TIME or TIMESTAMP timezone conversion. + // The timezone used by ORC reader/writer is a placeholder that is not used. + // Check OrcType::toOrcType() for reference. + else if (type.equals(TIME) || type.equals(TIMESTAMP)) { + storageType = BIGINT; + } + else if (type instanceof ArrayType) { + storageType = new ArrayType(toStorageType(((ArrayType) type).getElementType())); + } + else if (type instanceof MapType) { + storageType = mapType(toStorageType(((MapType) type).getKeyType()), toStorageType(((MapType) type).getValueType())); + } + else { + throw new PrestoException(NOT_SUPPORTED, "Type not supported: " + type); + } + + // We cannot write different java types because when Raptor calculates stats, it uses the column types rather than storage types. + // Need to make sure min/max are compliant with both storage and column types. + checkState(storageType.getJavaType().equals(type.getJavaType())); + return storageType; + } + + private Type mapType(Type keyType, Type valueType) + { + return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( + TypeSignatureParameter.of(keyType.getTypeSignature()), + TypeSignatureParameter.of(valueType.getTypeSignature()))); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java index 256cb42f66e74..6c99abfcd0dbd 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java @@ -16,7 +16,6 @@ import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.storage.ReaderAttributes; -import com.facebook.presto.raptor.storage.Row; import com.facebook.presto.raptor.storage.StorageManager; import com.facebook.presto.raptor.storage.StoragePageSink; import com.facebook.presto.spi.ConnectorPageSource; @@ -44,7 +43,6 @@ import java.util.Set; import java.util.UUID; -import static com.facebook.presto.raptor.storage.Row.extractRow; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.concurrent.MoreFutures.getFutureValue; import static io.airlift.units.Duration.nanosSince; @@ -128,23 +126,23 @@ public List compactSorted(long transactionId, OptionalInt bucketNumbe .map(columnIds::indexOf) .collect(toList()); - Queue rowSources = new PriorityQueue<>(); + Queue rowSources = new PriorityQueue<>(); StoragePageSink outputPageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false); try { for (UUID uuid : uuids) { ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes); - SortedRowSource rowSource = new SortedRowSource(pageSource, columnTypes, sortIndexes, sortOrders); + SortedPageSource rowSource = new SortedPageSource(pageSource, columnTypes, sortIndexes, sortOrders); rowSources.add(rowSource); } while (!rowSources.isEmpty()) { - SortedRowSource rowSource = rowSources.poll(); + SortedPageSource rowSource = rowSources.poll(); if (!rowSource.hasNext()) { // rowSource is empty, close it rowSource.close(); continue; } - outputPageSink.appendRow(rowSource.next()); + outputPageSink.appendPages(ImmutableList.of(rowSource.next())); if (outputPageSink.isFull()) { outputPageSink.flush(); @@ -164,12 +162,12 @@ public List compactSorted(long transactionId, OptionalInt bucketNumbe throw e; } finally { - rowSources.forEach(SortedRowSource::closeQuietly); + rowSources.forEach(SortedPageSource::closeQuietly); } } - private static class SortedRowSource - implements Iterator, Comparable, Closeable + private static class SortedPageSource + implements Iterator, Comparable, Closeable { private final ConnectorPageSource pageSource; private final List columnTypes; @@ -179,7 +177,7 @@ private static class SortedRowSource private Page currentPage; private int currentPosition; - public SortedRowSource(ConnectorPageSource pageSource, List columnTypes, List sortIndexes, List sortOrders) + public SortedPageSource(ConnectorPageSource pageSource, List columnTypes, List sortIndexes, List sortOrders) { this.pageSource = requireNonNull(pageSource, "pageSource is null"); this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); @@ -219,19 +217,20 @@ private static Page getNextPage(ConnectorPageSource pageSource) } @Override - public Row next() + public Page next() { if (!hasNext()) { throw new NoSuchElementException(); } - Row row = extractRow(currentPage, currentPosition, columnTypes); + int[] mask = {currentPosition}; + Page page = currentPage.getPositions(mask, 0, 1); currentPosition++; - return row; + return page; } @Override - public int compareTo(SortedRowSource other) + public int compareTo(SortedPageSource other) { if (!hasNext()) { return 1; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java index bd8426fffdc55..f7c453922d9d6 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java @@ -100,7 +100,7 @@ public void setup() NodeManager nodeManager = new TestingNodeManager(); NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; ShardManager shardManager = createShardManager(dbi); - StorageManager storageManager = createOrcStorageManager(dbi, dataDir); + StorageManager storageManager = createOrcStorageManager(dbi, dataDir, true); StorageManagerConfig config = new StorageManagerConfig(); connector = new RaptorConnector( new LifeCycleManager(ImmutableList.of(), null), diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueries.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java similarity index 89% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueries.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java index 39fe2ed023932..62467a93d98fe 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueries.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; import com.facebook.presto.tests.AbstractTestDistributedQueries; import com.google.common.collect.ImmutableMap; @@ -24,7 +24,7 @@ public class TestRaptorDistributedQueries @SuppressWarnings("unused") public TestRaptorDistributedQueries() { - this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false)); + this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); } protected TestRaptorDistributedQueries(QueryRunnerSupplier supplier) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueriesBucketed.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java similarity index 87% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueriesBucketed.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java index cd7aecf3a7435..e1dec16bc3a53 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueriesBucketed.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; import com.google.common.collect.ImmutableMap; @@ -22,7 +22,7 @@ public class TestRaptorDistributedQueriesBucketed { public TestRaptorDistributedQueriesBucketed() { - super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true)); + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); } @Override diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTest.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java similarity index 96% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTest.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java index 9378f647ff9d0..f5ddd50d515ac 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTest.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.testing.MaterializedResult; @@ -60,7 +60,7 @@ public class TestRaptorIntegrationSmokeTest @SuppressWarnings("unused") public TestRaptorIntegrationSmokeTest() { - this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false)); + this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "DISABLED"))); } protected TestRaptorIntegrationSmokeTest(QueryRunnerSupplier supplier) @@ -771,4 +771,28 @@ public void testAlterTable() assertUpdate("DROP TABLE test_alter_table"); } + + @Test + public void testDelete() + { + assertUpdate("CREATE TABLE test_delete_table (c1 bigint, c2 bigint)"); + assertUpdate("INSERT INTO test_delete_table VALUES (1, 1), (1, 2), (1, 3), (1, 4), (11, 1), (11, 2)", 6); + + assertUpdate("ALTER TABLE test_delete_table ADD COLUMN c3 bigint"); + assertUpdate("INSERT INTO test_delete_table VALUES (2, 1, 1), (2, 2, 2), (2, 3, 3), (2, 4, 4), (22, 1, 1), (22, 2, 2), (22, 4, 4)", 7); + + assertUpdate("DELETE FROM test_delete_table WHERE c1 = 1", 4); + assertQuery("SELECT * FROM test_delete_table", "VALUES (11, 1, NULL), (11, 2, NULL), (2, 1, 1), (2, 2, 2), (2, 3, 3), (2, 4, 4), (22, 1, 1), (22, 2, 2), (22, 4, 4)"); + + assertUpdate("ALTER TABLE test_delete_table DROP COLUMN c2"); + assertUpdate("INSERT INTO test_delete_table VALUES (3, 1), (3, 2), (3, 3), (3, 4)", 4); + + assertUpdate("DELETE FROM test_delete_table WHERE c1 = 2", 4); + assertQuery("SELECT * FROM test_delete_table", "VALUES (11, NULL), (11, NULL), (22, 1), (22, 2), (22, 4), (3, 1), (3, 2), (3, 3), (3, 4)"); + + assertUpdate("DELETE FROM test_delete_table WHERE c1 % 11 = 0", 5); + assertQuery("SELECT * FROM test_delete_table", "VALUES (3, 1), (3, 2), (3, 3), (3, 4)"); + + assertUpdate("DROP TABLE test_delete_table"); + } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestBucketed.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java similarity index 91% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestBucketed.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java index 88716b37d481d..ef58207d9dc73 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestBucketed.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -23,7 +23,7 @@ public class TestRaptorIntegrationSmokeTestBucketed { public TestRaptorIntegrationSmokeTestBucketed() { - super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true)); + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "DISABLED"))); } @Test diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java new file mode 100644 index 0000000000000..d2b1b8597fd3e --- /dev/null +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.integration; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.raptor.RaptorQueryRunner.createRaptorQueryRunner; + +public class TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter + extends TestRaptorIntegrationSmokeTest +{ + public TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter() + { + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); + } + + @Test + public void testShardsSystemTableBucketNumber() + { + assertQuery("" + + "SELECT count(DISTINCT bucket_number)\n" + + "FROM system.shards\n" + + "WHERE table_schema = 'tpch'\n" + + " AND table_name = 'orders'", + "SELECT 25"); + } +} diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestMySql.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java similarity index 96% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestMySql.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java index bd9671c224a70..595824864e29e 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestMySql.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java @@ -11,8 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; +import com.facebook.presto.raptor.RaptorPlugin; import com.facebook.presto.tests.DistributedQueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java new file mode 100644 index 0000000000000..28eff3d093f29 --- /dev/null +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.integration; + +import com.google.common.collect.ImmutableMap; + +import static com.facebook.presto.raptor.RaptorQueryRunner.createRaptorQueryRunner; + +public class TestRaptorIntegrationSmokeTestWithOptimizedWriter + extends TestRaptorIntegrationSmokeTest +{ + @SuppressWarnings("unused") + public TestRaptorIntegrationSmokeTestWithOptimizedWriter() + { + this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); + } + + protected TestRaptorIntegrationSmokeTestWithOptimizedWriter(QueryRunnerSupplier supplier) + { + super(supplier); + } +} diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java index 6c2c0a52d48a0..76453d31bac7e 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java @@ -14,7 +14,7 @@ package com.facebook.presto.raptor.metadata; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; @@ -97,8 +97,8 @@ public void testNotEnoughNodes() TestingTicker ticker = new TestingTicker(); HashSet nodes = new HashSet<>(); - Node node1 = new PrestoNode("node1", new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); - Node node2 = new PrestoNode("node2", new URI("http://127.0.0.2/"), NodeVersion.UNKNOWN, false); + Node node1 = new InternalNode("node1", new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); + Node node2 = new InternalNode("node2", new URI("http://127.0.0.2/"), NodeVersion.UNKNOWN, false); nodes.add(node1); nodes.add(node2); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java index d67fa5e43cfed..13a0a21702463 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java @@ -14,7 +14,7 @@ package com.facebook.presto.raptor.metadata; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.util.DaoSupplier; @@ -821,7 +821,7 @@ private static Set nodeIds(Collection nodes) private static Node createTestingNode() { - return new PrestoNode(UUID.randomUUID().toString(), URI.create("http://test"), NodeVersion.UNKNOWN, false); + return new InternalNode(UUID.randomUUID().toString(), URI.create("http://test"), NodeVersion.UNKNOWN, false); } private int columnCount(long tableId) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java index d084a1800ec85..e47c53492bf28 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java @@ -14,8 +14,8 @@ package com.facebook.presto.raptor.metadata; import com.facebook.presto.client.NodeVersion; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.MetadataUtil.TableMetadataBuilder; -import com.facebook.presto.metadata.PrestoNode; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.RaptorConnectorId; @@ -105,7 +105,7 @@ public void setup() NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; String nodeName = UUID.randomUUID().toString(); - nodeManager.addNode(new PrestoNode(nodeName, new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false)); + nodeManager.addNode(new InternalNode(nodeName, new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false)); RaptorConnectorId connectorId = new RaptorConnectorId("raptor"); metadata = new RaptorMetadata(connectorId.toString(), dbi, shardManager); @@ -174,7 +174,7 @@ public void testAssignRandomNodeWhenBackupAvailable() TestingNodeManager nodeManager = new TestingNodeManager(); RaptorConnectorId connectorId = new RaptorConnectorId("raptor"); NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - PrestoNode node = new PrestoNode(UUID.randomUUID().toString(), new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); + InternalNode node = new InternalNode(UUID.randomUUID().toString(), new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); nodeManager.addNode(node); RaptorSplitManager raptorSplitManagerWithBackup = new RaptorSplitManager(connectorId, nodeSupplier, shardManager, true); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java index 7b3da44b5f38f..a8d5600e0a8ac 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java @@ -13,12 +13,17 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.orc.FileOrcDataSource; import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableMap; import io.airlift.units.DataSize; import org.joda.time.DateTimeZone; @@ -33,6 +38,8 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static org.testng.Assert.assertEquals; @@ -96,4 +103,14 @@ public static byte octet(int b) checkArgument((b >= 0) && (b <= 0xFF), "octet not in range: %s", b); return (byte) b; } + + public static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean useOptimizedOrcWriter) + { + if (useOptimizedOrcWriter) { + TypeRegistry typeManager = new TypeRegistry(); + new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + return new OrcFileWriter(columnIds, columnTypes, file, true, true, new OrcWriterStats(), typeManager, ZSTD); + } + return new OrcRecordWriter(columnIds, columnTypes, file, SNAPPY, true); + } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java index 416ce8753e1ce..f7e4281f79a4e 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java @@ -14,7 +14,7 @@ package com.facebook.presto.raptor.storage; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.metadata.BucketNode; import com.facebook.presto.raptor.metadata.ColumnInfo; @@ -306,6 +306,6 @@ private static BucketNode bucketNode(int bucketNumber, String nodeIdentifier) private static Node createTestingNode(String nodeIdentifier) { - return new PrestoNode(nodeIdentifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); + return new InternalNode(nodeIdentifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java index af99c73b61b04..b6d7ab0d1f106 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java @@ -16,10 +16,14 @@ import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.orc.OrcReader; import com.facebook.presto.orc.OrcRecordReader; -import com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo; +import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.raptor.metadata.TableColumn; +import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.StandardTypes; @@ -31,22 +35,38 @@ import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; import io.airlift.json.JsonCodec; +import io.airlift.units.DataSize; +import org.skife.jdbi.v2.DBI; +import org.skife.jdbi.v2.Handle; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; import java.math.BigDecimal; import java.util.BitSet; import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; +import static com.facebook.presto.orc.OrcEncoding.ORC; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; +import static com.facebook.presto.raptor.storage.FileStorageService.getFileSystemPath; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.fileOrcDataSource; +import static com.facebook.presto.raptor.storage.TestOrcStorageManager.createOrcStorageManager; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; import static com.facebook.presto.tests.StructuralTestUtil.arrayBlockOf; @@ -58,19 +78,29 @@ import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.testing.Assertions.assertBetweenInclusive; +import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.nio.file.Files.readAllBytes; import static java.util.UUID.randomUUID; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; @Test(singleThreaded = true) public class TestOrcFileRewriter { + private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); private File temporary; + @DataProvider(name = "useOptimizedOrcWriter") + public static Object[][] useOptimizedOrcWriter() + { + return new Object[][] {{true}, {false}}; + } + @BeforeClass public void setup() { @@ -84,8 +114,8 @@ public void tearDown() deleteRecursively(temporary.toPath(), ALLOW_INSECURE); } - @Test - public void testRewrite() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewrite(boolean useOptimizedOrcWriter) throws Exception { TypeManager typeManager = new TypeRegistry(); @@ -103,7 +133,7 @@ public void testRewrite() List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), arrayType, mapType, arrayOfArrayType, decimalType); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { List pages = rowPagesBuilder(columnTypes) .row(123L, "hello", arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), new BigDecimal("2.3")) .row(777L, "sky", arrayBlockOf(BIGINT, 3, 4), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)), new BigDecimal("2.3")) @@ -197,9 +227,9 @@ public void testRewrite() rowsToDelete.set(4); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, rowsToDelete); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); assertEquals(info.getRowCount(), 2); - assertEquals(info.getUncompressedSize(), 94); + assertBetweenInclusive(info.getUncompressedSize(), 94L, 118L * 2); try (OrcDataSource dataSource = fileOrcDataSource(newFile)) { OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); @@ -264,15 +294,15 @@ public void testRewrite() } } - @Test - public void testRewriteWithoutMetadata() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) throws Exception { List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file, false)) { + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { List pages = rowPagesBuilder(columnTypes) .row(123L, "hello") .row(777L, "sky") @@ -312,9 +342,9 @@ public void testRewriteWithoutMetadata() rowsToDelete.set(1); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, rowsToDelete); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); assertEquals(info.getRowCount(), 1); - assertEquals(info.getUncompressedSize(), 13); + assertBetweenInclusive(info.getUncompressedSize(), 13L, 13L * 2); try (OrcDataSource dataSource = fileOrcDataSource(newFile)) { OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); @@ -339,15 +369,15 @@ public void testRewriteWithoutMetadata() } } - @Test - public void testRewriteAllRowsDeleted() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriteAllRowsDeleted(boolean useOptimizedOrcWriter) throws Exception { List columnIds = ImmutableList.of(3L); List columnTypes = ImmutableList.of(BIGINT); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); } @@ -356,44 +386,43 @@ public void testRewriteAllRowsDeleted() rowsToDelete.set(1); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, rowsToDelete); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); assertEquals(info.getRowCount(), 0); assertEquals(info.getUncompressedSize(), 0); assertFalse(newFile.exists()); } - @Test - public void testRewriteNoRowsDeleted() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriteNoRowsDeleted(boolean useOptimizedOrcWriter) throws Exception { List columnIds = ImmutableList.of(3L); List columnTypes = ImmutableList.of(BIGINT); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); } BitSet rowsToDelete = new BitSet(); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, rowsToDelete); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); assertEquals(info.getRowCount(), 2); - assertEquals(info.getUncompressedSize(), 16); - + assertBetweenInclusive(info.getUncompressedSize(), 16L, 16L * 2); assertEquals(readAllBytes(newFile.toPath()), readAllBytes(file.toPath())); } - @Test - public void testUncompressedSize() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testUncompressedSize(boolean useOptimizedOrcWriter) throws Exception { List columnIds = ImmutableList.of(1L, 2L, 3L, 4L, 5L); List columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { List pages = rowPagesBuilder(columnTypes) .row(true, 123L, 98.7, "hello", utf8Slice("abc")) .row(false, 456L, 65.4, "world", utf8Slice("xyz")) @@ -403,8 +432,280 @@ public void testUncompressedSize() } File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, new BitSet()); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, new BitSet()); assertEquals(info.getRowCount(), 3); - assertEquals(info.getUncompressedSize(), 55); + assertBetweenInclusive(info.getUncompressedSize(), 55L, 55L * 2); + } + + /** + * The following test add or drop different columns + */ + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriterDropThenAddDifferentColumns(boolean useOptimizedOrcWriter) + throws Exception + { + TypeRegistry typeRegistry = new TypeRegistry(); + DBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); + dbi.registerMapper(new TableColumn.Mapper(typeRegistry)); + Handle dummyHandle = dbi.open(); + File dataDir = Files.createTempDir(); + + StorageManager storageManager = createOrcStorageManager(dbi, dataDir, useOptimizedOrcWriter); + + List columnIds = ImmutableList.of(3L, 7L); + List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); + + File file = new File(temporary, randomUUID().toString()); + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { + List pages = rowPagesBuilder(columnTypes) + .row(1L, "1") + .row(2L, "2") + .row(3L, "3") + .row(4L, "4") + .build(); + writer.appendPages(pages); + } + + // Add a column + File newFile1 = new File(temporary, randomUUID().toString()); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(3L, 7L, 10L), ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE)), + file, + newFile1, + new BitSet(5)); + assertEquals(info.getRowCount(), 4); + assertEquals(readAllBytes(file.toPath()), readAllBytes(newFile1.toPath())); + + // Drop a column + File newFile2 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 10L), ImmutableList.of(createVarcharType(20), DOUBLE)), + newFile1, + newFile2, + new BitSet(5)); + assertEquals(info.getRowCount(), 4); + if (useOptimizedOrcWriter) { + // Optimized writer will keep the only column + OrcReader orcReader = new OrcReader(fileOrcDataSource(newFile2), ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + orcReader.getColumnNames().equals(ImmutableList.of("7")); + } + else { + // Apache writer does not drop columns + assertEquals(readAllBytes(newFile1.toPath()), readAllBytes(newFile2.toPath())); + } + + // Add a column with the different ID with different type + File newFile3 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 10L, 13L), ImmutableList.of(createVarcharType(20), DOUBLE, createVarcharType(5))), + newFile2, + newFile3, + new BitSet(5)); + assertEquals(info.getRowCount(), 4); + assertEquals(readAllBytes(newFile2.toPath()), readAllBytes(newFile3.toPath())); + + // Get prepared for the final file; make sure it is accessible from storage manager + UUID uuid = randomUUID(); + File newFile4 = getFileSystemPath(new File(dataDir, "data/storage"), uuid); + if (useOptimizedOrcWriter) { + // Apache ORC writer creates the file itself + newFile4.getParentFile().mkdirs(); + newFile4.createNewFile(); + } + + // Drop a column and add a column; also delete 3 rows + BitSet rowsToDelete = new BitSet(5); + rowsToDelete.set(0); + rowsToDelete.set(1); + rowsToDelete.set(3); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 13L, 18L), ImmutableList.of(createVarcharType(20), createVarcharType(5), INTEGER)), + newFile3, + newFile4, + rowsToDelete); + assertEquals(info.getRowCount(), 1); + + ConnectorPageSource source = storageManager.getPageSource( + uuid, + OptionalInt.empty(), + ImmutableList.of(13L, 7L, 18L), + ImmutableList.of(createVarcharType(5), createVarcharType(20), INTEGER), + TupleDomain.all(), + READER_ATTRIBUTES); + + Page page = null; + while (page == null) { + page = source.getNextPage(); + } + assertEquals(page.getPositionCount(), 1); + + // Column 13L + Block column0 = page.getBlock(0); + assertTrue(column0.isNull(0)); + + // Column 7L + Block column1 = page.getBlock(1); + assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("3")); + + // Column 8L + Block column2 = page.getBlock(2); + assertTrue(column2.isNull(0)); + + // Remove all the columns + File newFile5 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(13L, 18L), ImmutableList.of(createVarcharType(5), INTEGER)), + newFile4, + newFile5, + new BitSet(5)); + if (useOptimizedOrcWriter) { + // Optimized writer will drop the file + assertEquals(info.getRowCount(), 0); + assertFalse(newFile5.exists()); + } + else { + assertEquals(info.getRowCount(), 1); + assertEquals(readAllBytes(newFile4.toPath()), readAllBytes(newFile5.toPath())); + } + + dummyHandle.close(); + deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); + } + + /** + * The following test drop and add the same columns; the legacy ORC rewriter will fail due to unchanged schema. + * However, if we enforce the newly added column to always have the largest ID, this won't happen. + */ + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriterDropThenAddSameColumns(boolean useOptimizedOrcWriter) + throws Exception + { + TypeRegistry typeRegistry = new TypeRegistry(); + DBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); + dbi.registerMapper(new TableColumn.Mapper(typeRegistry)); + Handle dummyHandle = dbi.open(); + File dataDir = Files.createTempDir(); + + StorageManager storageManager = createOrcStorageManager(dbi, dataDir, useOptimizedOrcWriter); + + List columnIds = ImmutableList.of(3L, 7L); + List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); + + File file = new File(temporary, randomUUID().toString()); + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { + List pages = rowPagesBuilder(columnTypes) + .row(2L, "2") + .build(); + writer.appendPages(pages); + } + + // Add a column + File newFile1 = new File(temporary, randomUUID().toString()); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(3L, 7L, 10L), ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE)), + file, + newFile1, + new BitSet(5)); + assertEquals(info.getRowCount(), 1); + + // Drop a column + File newFile2 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 10L), ImmutableList.of(createVarcharType(20), DOUBLE)), + newFile1, + newFile2, + new BitSet(5)); + assertEquals(info.getRowCount(), 1); + + // Add a column with the same ID but different type + File newFile3 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 10L, 3L), ImmutableList.of(createVarcharType(20), DOUBLE, createVarcharType(5))), + newFile2, + newFile3, + new BitSet(5)); + assertEquals(info.getRowCount(), 1); + + // Get prepared for the final file; make sure it is accessible from storage manager + UUID uuid = randomUUID(); + File newFile4 = getFileSystemPath(new File(dataDir, "data/storage"), uuid); + if (useOptimizedOrcWriter) { + // Apache ORC writer creates the file itself + newFile4.getParentFile().mkdirs(); + newFile4.createNewFile(); + } + + // Drop a column and add a column + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 3L, 8L), ImmutableList.of(createVarcharType(20), createVarcharType(5), INTEGER)), + newFile3, + newFile4, + new BitSet(5)); + assertEquals(info.getRowCount(), 1); + + ConnectorPageSource source = storageManager.getPageSource( + uuid, + OptionalInt.empty(), + ImmutableList.of(3L, 7L, 8L), + ImmutableList.of(createVarcharType(5), createVarcharType(20), INTEGER), + TupleDomain.all(), + READER_ATTRIBUTES); + + Page page = null; + while (page == null) { + page = source.getNextPage(); + } + assertEquals(page.getPositionCount(), 1); + + try { + // Column 3L + Block column0 = page.getBlock(0); + assertTrue(column0.isNull(0)); + + // Column 7L + Block column1 = page.getBlock(1); + assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("2")); + + // Column 8L + Block column2 = page.getBlock(2); + assertTrue(column2.isNull(0)); + + dummyHandle.close(); + deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); + + if (!useOptimizedOrcWriter) { + // Apache ORC rewriter will not respect the schema + fail(); + } + } + catch (UnsupportedOperationException e) { + if (useOptimizedOrcWriter) { + // Optimized ORC rewriter will respect the schema + fail(); + } + } + } + + private static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean writeMetadata, boolean useOptimizedOrcWriter) + { + if (useOptimizedOrcWriter) { + return new OrcFileWriter(columnIds, columnTypes, file, writeMetadata, true, new OrcWriterStats(), new TypeRegistry(), ZSTD); + } + return new OrcRecordWriter(columnIds, columnTypes, file, SNAPPY, writeMetadata); + } + + private static FileRewriter createFileRewriter(boolean useOptimizedOrcWriter) + { + if (useOptimizedOrcWriter) { + TypeRegistry typeManager = new TypeRegistry(); + new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + return new OrcPageFileRewriter(READER_ATTRIBUTES, true, new OrcWriterStats(), typeManager, ZSTD); + } + return new OrcRecordFileRewriter(); + } + + private static Map getColumnTypes(List columnIds, List columnTypes) + { + return IntStream.range(0, columnIds.size()).boxed().collect(Collectors.toMap(index -> String.valueOf(columnIds.get(index)), columnTypes::get)); } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java index a64f9c52027ca..0b488544e29cd 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java @@ -32,6 +32,7 @@ import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.SqlDate; +import com.facebook.presto.spi.type.SqlTime; import com.facebook.presto.spi.type.SqlTimestamp; import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.Type; @@ -52,6 +53,7 @@ import org.skife.jdbi.v2.IDBI; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; @@ -65,17 +67,23 @@ import java.util.OptionalLong; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.metadata.TestDatabaseShardManager.createShardManager; import static com.facebook.presto.raptor.storage.OrcStorageManager.xxhash64; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.octets; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.TimeType.TIME; import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; @@ -97,6 +105,7 @@ import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.joda.time.DateTimeZone.UTC; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; @@ -130,6 +139,12 @@ public class TestOrcStorageManager private Optional backupStore; private InMemoryShardRecorder shardRecorder; + @DataProvider(name = "useOptimizedOrcWriter") + public static Object[][] useOptimizedOrcWriter() + { + return new Object[][] {{true}, {false}}; + } + @BeforeMethod public void setup() { @@ -164,11 +179,11 @@ public void tearDown() deleteRecursively(temporary.toPath(), ALLOW_INSECURE); } - @Test - public void testWriter() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testWriter(boolean useOptimizedOrcWriter) throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -237,11 +252,11 @@ public void testWriter() } } - @Test - public void testReader() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testReader(boolean useOptimizedOrcWriter) throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); List columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE); @@ -311,11 +326,11 @@ public void testReader() } } - @Test - public void testRewriter() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriter(boolean useOptimizedOrcWriter) throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); long transactionId = TRANSACTION_ID; List columnIds = ImmutableList.of(3L, 7L); @@ -335,7 +350,12 @@ public void testRewriter() // delete one row BitSet rowsToDelete = new BitSet(); rowsToDelete.set(0); - Collection fragments = manager.rewriteShard(transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete); + Collection fragments = manager.rewriteShard( + transactionId, + OptionalInt.empty(), + shards.get(0).getShardUuid(), + IntStream.range(0, columnIds.size()).boxed().collect(Collectors.toMap(index -> String.valueOf(columnIds.get(index)), columnTypes::get)), + rowsToDelete); Slice shardDelta = Iterables.getOnlyElement(fragments); ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes()); @@ -356,8 +376,8 @@ public void testRewriter() assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid()); } - @Test - public void testWriterRollback() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testWriterRollback(boolean useOptimizedOrcWriter) { // verify staging directory is empty File staging = new File(new File(temporary, "data"), "staging"); @@ -365,7 +385,7 @@ public void testWriterRollback() assertEquals(staging.list(), new String[] {}); // create a shard in staging - OrcStorageManager manager = createOrcStorageManager(); + OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -500,9 +520,20 @@ public void testShardStatsDateTimestamp() } @Test - public void testMaxShardRows() + public void testShardStatsTime() { - OrcStorageManager manager = createOrcStorageManager(2, new DataSize(2, MEGABYTE)); + long minTime = sqlTime(2004, 8, 22).getMillis(); + long maxTime = sqlTime(2006, 4, 22).getMillis(); + + // Apache ORC writer does not support TIME + List columnStats = columnStats(types(TIME), row(minTime), row(maxTime)); + assertColumnStats(columnStats, 1, minTime, maxTime); + } + + @Test(dataProvider = "useOptimizedOrcWriter") + public void testMaxShardRows(boolean useOptimizedOrcWriter) + { + OrcStorageManager manager = createOrcStorageManager(2, new DataSize(2, MEGABYTE), useOptimizedOrcWriter); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -516,8 +547,8 @@ public void testMaxShardRows() assertTrue(sink.isFull()); } - @Test - public void testMaxFileSize() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testMaxFileSize(boolean useOptimizedOrcWriter) { List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(5)); @@ -528,7 +559,7 @@ public void testMaxFileSize() .build(); // Set maxFileSize to 1 byte, so adding any page makes the StoragePageSink full - OrcStorageManager manager = createOrcStorageManager(20, new DataSize(1, BYTE)); + OrcStorageManager manager = createOrcStorageManager(20, new DataSize(1, BYTE), useOptimizedOrcWriter); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); sink.appendPages(pages); assertTrue(sink.isFull()); @@ -550,22 +581,22 @@ private static StoragePageSink createStoragePageSink(StorageManager manager, Lis return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); } - private OrcStorageManager createOrcStorageManager() + private OrcStorageManager createOrcStorageManager(boolean useOptimizedWriter) { - return createOrcStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE); + return createOrcStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE, useOptimizedWriter); } - private OrcStorageManager createOrcStorageManager(int maxShardRows, DataSize maxFileSize) + private OrcStorageManager createOrcStorageManager(int maxShardRows, DataSize maxFileSize, boolean useOptimizedWriter) { - return createOrcStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize); + return createOrcStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize, useOptimizedWriter); } - public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary) + public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, boolean useOptimizedWriter) { - return createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); + return createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedWriter); } - public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, int maxShardRows) + public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, int maxShardRows, boolean useOptimizedWriter) { File directory = new File(temporary, "data"); StorageService storageService = new FileStorageService(directory); @@ -590,7 +621,8 @@ public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary recoveryManager, new InMemoryShardRecorder(), maxShardRows, - MAX_FILE_SIZE); + MAX_FILE_SIZE, + useOptimizedWriter); } public static OrcStorageManager createOrcStorageManager( @@ -599,7 +631,8 @@ public static OrcStorageManager createOrcStorageManager( ShardRecoveryManager recoveryManager, ShardRecorder shardRecorder, int maxShardRows, - DataSize maxFileSize) + DataSize maxFileSize, + boolean useOptimizedWriter) { return new OrcStorageManager( CURRENT_NODE, @@ -615,7 +648,9 @@ public static OrcStorageManager createOrcStorageManager( SHARD_RECOVERY_TIMEOUT, maxShardRows, maxFileSize, - new DataSize(0, BYTE)); + new DataSize(0, BYTE), + SNAPPY, + useOptimizedWriter ? ENABLED_AND_VALIDATED : DISABLED); } private static void assertFileEquals(File actual, File expected) @@ -661,7 +696,15 @@ private List columnStats(List columnTypes, Object[]... rows) } List columnIds = list.build(); - OrcStorageManager manager = createOrcStorageManager(); + List apacheOrcWriterStats = columnStats(false, columnIds, columnTypes, rows); + List optimizedOrcWriterStats = columnStats(true, columnIds, columnTypes, rows); + assertEquals(apacheOrcWriterStats, optimizedOrcWriterStats); + return optimizedOrcWriterStats; + } + + private List columnStats(boolean useOptimizedWriter, List columnIds, List columnTypes, Object[]... rows) + { + OrcStorageManager manager = createOrcStorageManager(useOptimizedWriter); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); sink.appendPages(rowPagesBuilder(columnTypes).rows(rows).build()); List shards = getFutureValue(sink.commit()); @@ -681,6 +724,12 @@ private static SqlDate sqlDate(int year, int month, int day) return new SqlDate(Days.daysBetween(EPOCH, date).getDays()); } + private static SqlTime sqlTime(int year, int month, int day) + { + DateTime date = new DateTime(year, month, day, 0, 0, 0, 0, UTC); + return new SqlTime(NANOSECONDS.toMillis(date.toLocalTime().getMillisOfDay())); + } + private static SqlTimestamp sqlTimestamp(int year, int month, int day, int hour, int minute, int second) { return sqlTimestampOf(year, month, day, hour, minute, second, 0, UTC, UTC_KEY, SESSION); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java index 5e05cf242b732..2d34aa7d13197 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java @@ -14,7 +14,7 @@ package com.facebook.presto.raptor.storage; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.raptor.backup.BackupStore; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.MetadataDao; @@ -199,7 +199,7 @@ private static NodeManager createNodeManager(String current, String... others) private static Node createTestingNode(String identifier) { - return new PrestoNode(identifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); + return new InternalNode(identifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); } private static class TestingBackupStore diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java index db4ca32adb0a8..178aee64d00d8 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java @@ -18,9 +18,13 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.SqlDate; +import com.facebook.presto.spi.type.SqlTime; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -31,22 +35,34 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.json.JsonCodec; +import org.joda.time.DateTime; +import org.joda.time.Days; +import org.joda.time.chrono.ISOChronology; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; import java.util.List; +import static com.facebook.presto.raptor.storage.OrcTestingUtil.createFileWriter; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReaderNoRows; import static com.facebook.presto.raptor.storage.OrcTestingUtil.fileOrcDataSource; import static com.facebook.presto.raptor.storage.OrcTestingUtil.octets; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.TimeType.TIME; +import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; +import static com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.facebook.presto.tests.StructuralTestUtil.arrayBlockOf; import static com.facebook.presto.tests.StructuralTestUtil.arrayBlocksEqual; import static com.facebook.presto.tests.StructuralTestUtil.mapBlockOf; @@ -57,9 +73,13 @@ import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.slice.Slices.wrappedBuffer; +import static java.util.Locale.ENGLISH; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.joda.time.DateTimeZone.UTC; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; public class TestShardWriter { @@ -67,6 +87,12 @@ public class TestShardWriter private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); + @DataProvider(name = "useOptimizedOrcWriter") + public static Object[][] useOptimizedOrcWriter() + { + return new Object[][] {{true}, {false}}; + } + @BeforeClass public void setup() { @@ -80,33 +106,37 @@ public void tearDown() deleteRecursively(directory.toPath(), ALLOW_INSECURE); } - @Test - public void testWriter() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testWriter(boolean useOptimizedOrcWriter) throws Exception { TypeManager typeManager = new TypeRegistry(); // associate typeManager with a function manager new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - List columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L); + List columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L); ArrayType arrayType = new ArrayType(BIGINT); ArrayType arrayOfArrayType = new ArrayType(arrayType); Type mapType = typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( TypeSignatureParameter.of(createVarcharType(10).getTypeSignature()), TypeSignatureParameter.of(BOOLEAN.getTypeSignature()))); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType); + List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType, TIMESTAMP, TIME, DATE); File file = new File(directory, System.nanoTime() + ".orc"); byte[] bytes1 = octets(0x00, 0xFE, 0xFF); byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80); + long timestampValue = sqlTimestampOf(2002, 4, 6, 7, 8, 9, 0, UTC, UTC_KEY, SESSION).getMillisUtc(); + long timeValue = new SqlTime(NANOSECONDS.toMillis(new DateTime(2004, 11, 29, 0, 0, 0, 0, UTC).toLocalTime().getMillisOfDay())).getMillis(); + DateTime date = new DateTime(2001, 11, 22, 0, 0, 0, 0, UTC); + int dateValue = new SqlDate(Days.daysBetween(new DateTime(0, ISOChronology.getInstanceUTC()), date).getDays()).getDays(); RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(columnTypes) - .row(123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))) - .row(null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))) - .row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT))); + .row(123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), timestampValue, timeValue, dateValue) + .row(null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)), timestampValue, timeValue, dateValue) + .row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)), timestampValue, timeValue, dateValue); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader()); - OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + FileWriter writer = createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { writer.appendPages(rowPagesBuilder.build()); } @@ -176,6 +206,21 @@ public void testWriter() assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 1), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)))); assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)))); + Block column8 = reader.readBlock(TIMESTAMP, 8); + assertEquals(TIMESTAMP.getLong(column8, 0), timestampValue); + assertEquals(TIMESTAMP.getLong(column8, 1), timestampValue); + assertEquals(TIMESTAMP.getLong(column8, 2), timestampValue); + + Block column9 = reader.readBlock(TIME, 9); + assertEquals(TIME.getLong(column9, 0), timeValue); + assertEquals(TIME.getLong(column9, 1), timeValue); + assertEquals(TIME.getLong(column9, 2), timeValue); + + Block column10 = reader.readBlock(DATE, 10); + assertEquals(DATE.getLong(column10, 0), dateValue); + assertEquals(DATE.getLong(column10, 1), dateValue); + assertEquals(DATE.getLong(column10, 2), dateValue); + assertEquals(reader.nextBatch(), -1); assertEquals(reader.getReaderPosition(), 3); assertEquals(reader.getFilePosition(), reader.getFilePosition()); @@ -190,11 +235,25 @@ public void testWriter() .put(8L, arrayType.getTypeSignature()) .put(9L, mapType.getTypeSignature()) .put(10L, arrayOfArrayType.getTypeSignature()) + .put(11L, TIMESTAMP.getTypeSignature()) + .put(12L, TIME.getTypeSignature()) + .put(13L, DATE.getTypeSignature()) .build())); } File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc"); assertFalse(crcFile.exists()); + + // Test unsupported types + for (Type type : ImmutableList.of(TIMESTAMP_WITH_TIME_ZONE, RowType.anonymous(ImmutableList.of(BIGINT, DOUBLE)))) { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader())) { + createFileWriter(ImmutableList.of(1L), ImmutableList.of(type), file, useOptimizedOrcWriter); + fail(); + } + catch (PrestoException e) { + assertTrue(e.getMessage().toLowerCase(ENGLISH).contains("type")); + } + } } @SuppressWarnings("EmptyTryBlock") @@ -207,7 +266,7 @@ public void testWriterZeroRows() File file = new File(directory, System.nanoTime() + ".orc"); - try (OrcFileWriter ignored = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter ignored = createFileWriter(columnIds, columnTypes, file, false)) { // no rows } @@ -218,6 +277,19 @@ public void testWriterZeroRows() assertEquals(reader.nextBatch(), -1); } + + // optimized ORC writer will flush metadata on close + try (FileWriter ignored = createFileWriter(columnIds, columnTypes, file, true)) { + // no rows + } + + try (OrcDataSource dataSource = fileOrcDataSource(file)) { + OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); + assertEquals(reader.getReaderRowCount(), 0); + assertEquals(reader.getReaderPosition(), 0); + + assertEquals(reader.nextBatch(), -1); + } } @SuppressWarnings("EmptyClass") diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java index 6ee890e6fade6..d9dc2e5d5a3f9 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java @@ -24,6 +24,10 @@ import java.io.File; import java.util.Map; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; @@ -52,6 +56,8 @@ public void testDefaults() .setOrcStreamBufferSize(new DataSize(8, MEGABYTE)) .setOrcTinyStripeThreshold(new DataSize(8, MEGABYTE)) .setOrcLazyReadSmallRanges(true) + .setOrcOptimizedWriterStage(DISABLED) + .setOrcCompressionKind(SNAPPY) .setDeletionThreads(max(1, getRuntime().availableProcessors() / 2)) .setShardRecoveryTimeout(new Duration(30, SECONDS)) .setMissingShardDiscoveryInterval(new Duration(5, MINUTES)) @@ -81,6 +87,8 @@ public void testExplicitPropertyMappings() .put("storage.orc.stream-buffer-size", "16kB") .put("storage.orc.tiny-stripe-threshold", "15kB") .put("storage.orc.lazy-read-small-ranges", "false") + .put("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED") + .put("storage.orc.compression-kind", "ZSTD") .put("storage.max-deletion-threads", "999") .put("storage.shard-recovery-timeout", "1m") .put("storage.missing-shard-discovery-interval", "4m") @@ -107,6 +115,8 @@ public void testExplicitPropertyMappings() .setOrcStreamBufferSize(new DataSize(16, KILOBYTE)) .setOrcTinyStripeThreshold(new DataSize(15, KILOBYTE)) .setOrcLazyReadSmallRanges(false) + .setOrcOptimizedWriterStage(ENABLED_AND_VALIDATED) + .setOrcCompressionKind(ZSTD) .setDeletionThreads(999) .setShardRecoveryTimeout(new Duration(1, MINUTES)) .setMissingShardDiscoveryInterval(new Duration(4, MINUTES)) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java index 3a54275729037..9cc835ba8bca6 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java @@ -18,7 +18,6 @@ import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.ShardInfo; -import com.facebook.presto.raptor.storage.OrcStorageManager; import com.facebook.presto.raptor.storage.ReaderAttributes; import com.facebook.presto.raptor.storage.StorageManager; import com.facebook.presto.raptor.storage.StoragePageSink; @@ -38,6 +37,7 @@ import org.skife.jdbi.v2.IDBI; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; @@ -74,19 +74,22 @@ public class TestShardCompactor private static final PagesIndexPageSorter PAGE_SORTER = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - private OrcStorageManager storageManager; - private ShardCompactor compactor; private File temporary; + private IDBI dbi; private Handle dummyHandle; + @DataProvider(name = "useOptimizedOrcWriter") + public static Object[][] useOptimizedOrcWriter() + { + return new Object[][] {{true}, {false}}; + } + @BeforeMethod public void setup() { temporary = createTempDir(); - IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); + dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); dummyHandle = dbi.open(); - storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); - compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES); } @AfterMethod(alwaysRun = true) @@ -99,10 +102,11 @@ public void tearDown() deleteRecursively(temporary.toPath(), ALLOW_INSECURE); } - @Test - public void testShardCompactor() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testShardCompactor(boolean useOptimizedOrcWriter) throws Exception { + StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedOrcWriter); List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP); @@ -117,17 +121,19 @@ public void testShardCompactor() Set inputUuids = inputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); long transactionId = 1; + ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES); List outputShards = compactor.compact(transactionId, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes)); assertEquals(outputShards.size(), expectedOutputShards); Set outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); - assertShardEqualsIgnoreOrder(inputUuids, outputUuids, columnIds, columnTypes); + assertShardEqualsIgnoreOrder(storageManager, inputUuids, outputUuids, columnIds, columnTypes); } - @Test - public void testShardCompactorSorted() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testShardCompactorSorted(boolean useOptimizedOrcWriter) throws Exception { + StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedOrcWriter); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DATE, TIMESTAMP, DOUBLE); List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); List sortColumnIds = ImmutableList.of(1L, 2L, 3L, 5L, 7L); @@ -145,13 +151,14 @@ public void testShardCompactorSorted() Set inputUuids = inputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); long transactionId = 1; + ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES); List outputShards = compactor.compactSorted(transactionId, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes), sortColumnIds, sortOrders); List outputUuids = outputShards.stream() .map(ShardInfo::getShardUuid) .collect(toList()); assertEquals(outputShards.size(), expectedOutputShards); - assertShardEqualsSorted(inputUuids, outputUuids, columnIds, columnTypes, sortIndexes, sortOrders); + assertShardEqualsSorted(storageManager, inputUuids, outputUuids, columnIds, columnTypes, sortIndexes, sortOrders); } private static long computeExpectedOutputShards(long totalRows) @@ -159,23 +166,23 @@ private static long computeExpectedOutputShards(long totalRows) return ((totalRows % MAX_SHARD_ROWS) != 0) ? ((totalRows / MAX_SHARD_ROWS) + 1) : (totalRows / MAX_SHARD_ROWS); } - private void assertShardEqualsIgnoreOrder(Set inputUuids, Set outputUuids, List columnIds, List columnTypes) + private void assertShardEqualsIgnoreOrder(StorageManager storageManager, Set inputUuids, Set outputUuids, List columnIds, List columnTypes) throws IOException { - MaterializedResult inputRows = getMaterializedRows(ImmutableList.copyOf(inputUuids), columnIds, columnTypes); - MaterializedResult outputRows = getMaterializedRows(ImmutableList.copyOf(outputUuids), columnIds, columnTypes); + MaterializedResult inputRows = getMaterializedRows(storageManager, ImmutableList.copyOf(inputUuids), columnIds, columnTypes); + MaterializedResult outputRows = getMaterializedRows(storageManager, ImmutableList.copyOf(outputUuids), columnIds, columnTypes); assertEqualsIgnoreOrder(outputRows, inputRows); } - private void assertShardEqualsSorted(Set inputUuids, List outputUuids, List columnIds, List columnTypes, List sortIndexes, List sortOrders) + private void assertShardEqualsSorted(StorageManager storageManager, Set inputUuids, List outputUuids, List columnIds, List columnTypes, List sortIndexes, List sortOrders) throws IOException { - List inputPages = getPages(inputUuids, columnIds, columnTypes); + List inputPages = getPages(storageManager, inputUuids, columnIds, columnTypes); List sortTypes = sortIndexes.stream().map(columnTypes::get).collect(toList()); MaterializedResult inputRowsSorted = sortAndMaterialize(inputPages, columnTypes, sortIndexes, sortOrders, sortTypes); - MaterializedResult outputRows = extractColumns(getMaterializedRows(outputUuids, columnIds, columnTypes), sortIndexes, sortTypes); + MaterializedResult outputRows = extractColumns(getMaterializedRows(storageManager, outputUuids, columnIds, columnTypes), sortIndexes, sortTypes); assertEquals(outputRows, inputRowsSorted); } @@ -223,12 +230,12 @@ private static MaterializedResult sortAndMaterialize(List pages, List getPages(Set uuids, List columnIds, List columnTypes) + private List getPages(StorageManager storageManager, Set uuids, List columnIds, List columnTypes) throws IOException { ImmutableList.Builder pages = ImmutableList.builder(); for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = getPageSource(columnIds, columnTypes, uuid)) { + try (ConnectorPageSource pageSource = getPageSource(storageManager, columnIds, columnTypes, uuid)) { while (!pageSource.isFinished()) { Page outputPage = pageSource.getNextPage(); if (outputPage == null) { @@ -241,12 +248,12 @@ private List getPages(Set uuids, List columnIds, List co return pages.build(); } - private MaterializedResult getMaterializedRows(List uuids, List columnIds, List columnTypes) + private MaterializedResult getMaterializedRows(StorageManager storageManager, List uuids, List columnIds, List columnTypes) throws IOException { MaterializedResult.Builder rows = MaterializedResult.resultBuilder(SESSION, columnTypes); for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = getPageSource(columnIds, columnTypes, uuid)) { + try (ConnectorPageSource pageSource = getPageSource(storageManager, columnIds, columnTypes, uuid)) { MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); rows.rows(result.getMaterializedRows()); } @@ -254,7 +261,7 @@ private MaterializedResult getMaterializedRows(List uuids, List colu return rows.build(); } - private ConnectorPageSource getPageSource(List columnIds, List columnTypes, UUID uuid) + private ConnectorPageSource getPageSource(StorageManager storageManager, List columnIds, List columnTypes, UUID uuid) { return storageManager.getPageSource(uuid, OptionalInt.empty(), columnIds, columnTypes, TupleDomain.all(), READER_ATTRIBUTES); } diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 889629d26ad70..0f8d3658b2cc3 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -10,7 +10,6 @@ presto-server presto-server - provisio ${project.parent.basedir} @@ -24,4 +23,319 @@ com.facebook.presto.server.PrestoServer ${project.artifactId} + + + + + com.facebook.presto + presto-main + + + + + + io.airlift + launcher + ${dep.packaging.version} + bin + tar.gz + provided + + + + io.airlift + launcher + ${dep.packaging.version} + properties + tar.gz + provided + + + + + com.facebook.presto + presto-resource-group-managers + ${project.version} + zip + provided + + + + com.facebook.presto + presto-password-authenticators + ${project.version} + zip + provided + + + + com.facebook.presto + presto-session-property-managers + ${project.version} + zip + provided + + + + com.facebook.presto + presto-jmx + ${project.version} + zip + provided + + + + com.facebook.presto + presto-cassandra + ${project.version} + zip + provided + + + + com.facebook.presto + presto-example-http + ${project.version} + zip + provided + + + + com.facebook.presto + presto-hive-hadoop2 + ${project.version} + zip + provided + + + + com.facebook.presto + presto-memory + ${project.version} + zip + provided + + + + com.facebook.presto + presto-blackhole + ${project.version} + zip + provided + + + + com.facebook.presto + presto-kafka + ${project.version} + zip + provided + + + + com.facebook.presto + presto-kudu + ${project.version} + zip + provided + + + + com.facebook.presto + presto-atop + ${project.version} + zip + provided + + + + com.facebook.presto + presto-ml + ${project.version} + zip + provided + + + + com.facebook.presto + presto-geospatial + ${project.version} + zip + provided + + + + com.facebook.presto + presto-mysql + ${project.version} + zip + provided + + + + com.facebook.presto + presto-postgresql + ${project.version} + zip + provided + + + + com.facebook.presto + presto-redshift + ${project.version} + zip + provided + + + + com.facebook.presto + presto-sqlserver + ${project.version} + zip + provided + + + + com.facebook.presto + presto-raptor + ${project.version} + zip + provided + + + + com.facebook.presto + presto-redis + ${project.version} + zip + provided + + + + com.facebook.presto + presto-tpch + ${project.version} + zip + provided + + + + com.facebook.presto + presto-tpcds + ${project.version} + zip + provided + + + + com.facebook.presto + presto-teradata-functions + ${project.version} + zip + provided + + + + com.facebook.presto + presto-mongodb + ${project.version} + zip + provided + + + + com.facebook.presto + presto-local-file + ${project.version} + zip + provided + + + + com.facebook.presto + presto-accumulo + ${project.version} + zip + provided + + + + com.facebook.presto + presto-thrift-connector + ${project.version} + zip + provided + + + + com.facebook.presto + presto-elasticsearch + ${project.version} + zip + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-plugins + prepare-package + + unpack-dependencies + + + false + + zip + provided + + + + unpack-launcher + prepare-package + + unpack-dependencies + + + false + launcher + provided + ${project.build.directory}/dependency/launcher + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + bin + package + + single + + + + dir + tar.gz + + + src/main/assembly/presto.xml + + presto-server-${project.version} + false + + + + + + diff --git a/presto-server/src/main/assembly/presto.xml b/presto-server/src/main/assembly/presto.xml new file mode 100644 index 0000000000000..efd14285cc6c8 --- /dev/null +++ b/presto-server/src/main/assembly/presto.xml @@ -0,0 +1,168 @@ + + presto-server + true + + + + README.txt + + + NOTICE + + + + + + + false + runtime + lib + true + + + + + + + ${project.build.directory}/dependency/launcher/bin + + launcher + launcher.py + launcher.properties + + bin + + + ${project.build.directory}/dependency/launcher/bin + + launcher.properties + + bin + true + + + ${project.build.directory}/dependency/launcher/bin + + launcher + launcher.py + + bin + 0755 + + + + + ${project.build.directory}/dependency/presto-resource-group-managers-${project.version} + plugin/resource-group-managers + + + ${project.build.directory}/dependency/presto-password-authenticators-${project.version} + plugin/password-authenticators + + + ${project.build.directory}/dependency/presto-session-property-managers-${project.version} + plugin/session-property-managers + + + ${project.build.directory}/dependency/presto-jmx-${project.version} + plugin/jmx + + + ${project.build.directory}/dependency/presto-cassandra-${project.version} + plugin/cassandra + + + ${project.build.directory}/dependency/presto-example-http-${project.version} + plugin/example-http + + + ${project.build.directory}/dependency/presto-hive-hadoop2-${project.version} + plugin/hive-hadoop2 + + + ${project.build.directory}/dependency/presto-memory-${project.version} + plugin/memory + + + ${project.build.directory}/dependency/presto-blackhole-${project.version} + plugin/blackhole + + + ${project.build.directory}/dependency/presto-kafka-${project.version} + plugin/kafka + + + ${project.build.directory}/dependency/presto-kudu-${project.version} + plugin/kudu + + + ${project.build.directory}/dependency/presto-atop-${project.version} + plugin/atop + + + ${project.build.directory}/dependency/presto-ml-${project.version} + plugin/ml + + + ${project.build.directory}/dependency/presto-geospatial-${project.version} + plugin/geospatial + + + ${project.build.directory}/dependency/presto-mysql-${project.version} + plugin/mysql + + + ${project.build.directory}/dependency/presto-postgresql-${project.version} + plugin/postgresql + + + ${project.build.directory}/dependency/presto-redshift-${project.version} + plugin/redshift + + + ${project.build.directory}/dependency/presto-sqlserver-${project.version} + plugin/sqlserver + + + ${project.build.directory}/dependency/presto-raptor-${project.version} + plugin/raptor + + + ${project.build.directory}/dependency/presto-redis-${project.version} + plugin/redis + + + ${project.build.directory}/dependency/presto-tpch-${project.version} + plugin/tpch + + + ${project.build.directory}/dependency/presto-tpcds-${project.version} + plugin/tpcds + + + ${project.build.directory}/dependency/presto-teradata-functions-${project.version} + plugin/teradata-functions + + + ${project.build.directory}/dependency/presto-mongodb-${project.version} + plugin/mongodb + + + ${project.build.directory}/dependency/presto-local-file-${project.version} + plugin/localfile + + + ${project.build.directory}/dependency/presto-accumulo-${project.version} + plugin/accumulo + + + ${project.build.directory}/dependency/presto-thrift-connector-${project.version} + plugin/presto-thrift + + + ${project.build.directory}/dependency/presto-elasticsearch-${project.version} + plugin/presto-elasticsearch + + + diff --git a/presto-server/src/main/java/com/facebook/presto/Dummy.java b/presto-server/src/main/java/com/facebook/presto/Dummy.java new file mode 100644 index 0000000000000..363367c5c9f66 --- /dev/null +++ b/presto-server/src/main/java/com/facebook/presto/Dummy.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto; + +/** + * This class exists to force the creation of a jar for the presto-server module. This is needed to deploy the presto-server module to nexus. + */ +public class Dummy +{ +} diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml deleted file mode 100644 index a9f95e7c45c5a..0000000000000 --- a/presto-server/src/main/provisio/presto.xml +++ /dev/null @@ -1,196 +0,0 @@ - - - - - - - - NOTICE - README.txt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Node.java b/presto-spi/src/main/java/com/facebook/presto/spi/Node.java index 1f8d7591f3542..760ad224693e5 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Node.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Node.java @@ -17,8 +17,14 @@ public interface Node { + String getHost(); + HostAddress getHostAndPort(); + /** + * @deprecated Connectors should not access the HTTP endpoints of other nodes. + */ + @Deprecated URI getHttpUri(); String getNodeIdentifier(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Subfield.java b/presto-spi/src/main/java/com/facebook/presto/spi/Subfield.java new file mode 100644 index 0000000000000..6582925a659a9 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Subfield.java @@ -0,0 +1,300 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; + +public class Subfield +{ + public interface PathElement + { + boolean isSubscript(); + } + + public static final class AllSubscripts + implements PathElement + { + private static final AllSubscripts ALL_SUBSCRIPTS = new AllSubscripts(); + + private AllSubscripts() {} + + public static AllSubscripts getInstance() + { + return ALL_SUBSCRIPTS; + } + + @Override + public boolean isSubscript() + { + return true; + } + + @Override + public String toString() + { + return "[*]"; + } + } + + public static final class NestedField + implements PathElement + { + private final String name; + + public NestedField(String name) + { + this.name = requireNonNull(name, "name is null"); + } + + public String getName() + { + return name; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + NestedField that = (NestedField) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() + { + return Objects.hash(name); + } + + @Override + public String toString() + { + return "." + name; + } + + @Override + public boolean isSubscript() + { + return false; + } + } + + public static final class LongSubscript + implements PathElement + { + private final long index; + + public LongSubscript(long index) + { + this.index = index; + } + + public long getIndex() + { + return index; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + LongSubscript that = (LongSubscript) o; + return index == that.index; + } + + @Override + public int hashCode() + { + return Objects.hash(index); + } + + @Override + public String toString() + { + return "[" + index + "]"; + } + + @Override + public boolean isSubscript() + { + return true; + } + } + + public static final class StringSubscript + implements PathElement + { + private final String index; + + public StringSubscript(String index) + { + this.index = requireNonNull(index, "index is null"); + } + + public String getIndex() + { + return index; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + StringSubscript that = (StringSubscript) o; + return Objects.equals(index, that.index); + } + + @Override + public int hashCode() + { + return Objects.hash(index); + } + + @Override + public String toString() + { + return "[\"" + index.replace("\"", "\\\"") + "\"]"; + } + + @Override + public boolean isSubscript() + { + return true; + } + } + + private final String name; + private final List path; + + public static PathElement allSubscripts() + { + return AllSubscripts.getInstance(); + } + + @JsonCreator + public Subfield(String path) + { + requireNonNull(path, "path is null"); + + SubfieldTokenizer tokenizer = new SubfieldTokenizer(path); + checkArgument(tokenizer.hasNext(), "Column name is missing: " + path); + + PathElement firstElement = tokenizer.next(); + checkArgument(firstElement instanceof NestedField, "Subfield path must start with a name: " + path); + + this.name = ((NestedField) firstElement).getName(); + + List pathElements = new ArrayList<>(); + tokenizer.forEachRemaining(pathElements::add); + this.path = Collections.unmodifiableList(pathElements); + } + + private static void checkArgument(boolean expression, String errorMessage) + { + if (!expression) { + throw new IllegalArgumentException(errorMessage); + } + } + + public Subfield(String name, List path) + { + this.name = requireNonNull(name, "name is null"); + this.path = requireNonNull(path, "path is null"); + } + + public String getRootName() + { + return name; + } + + public List getPath() + { + return path; + } + + public boolean isPrefix(Subfield other) + { + if (!other.name.equals(name)) { + return false; + } + + if (path.size() < other.path.size()) { + return Objects.equals(path, other.path.subList(0, path.size())); + } + + return false; + } + + @JsonValue + public String serialize() + { + return name + path.stream() + .map(PathElement::toString) + .collect(Collectors.joining()); + } + + @Override + public String toString() + { + return serialize(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + Subfield other = (Subfield) o; + return Objects.equals(name, other.name) && + Objects.equals(path, other.path); + } + + @Override + public int hashCode() + { + return Objects.hash(name, path); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/SubfieldTokenizer.java b/presto-spi/src/main/java/com/facebook/presto/spi/SubfieldTokenizer.java new file mode 100644 index 0000000000000..3c78b73e0e9d8 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/SubfieldTokenizer.java @@ -0,0 +1,284 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static java.lang.Character.isLetterOrDigit; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +class SubfieldTokenizer + implements Iterator +{ + private static final char QUOTE = '\"'; + private static final char BACKSLASH = '\\'; + private static final char DOT = '.'; + private static final char OPEN_BRACKET = '['; + private static final char CLOSE_BRACKET = ']'; + private static final char UNICODE_CARET = '\u2038'; + private static final char WILDCARD = '*'; + + private final String path; + private State state = State.NOT_READY; + private int index; + private boolean firstSegment = true; + private Subfield.PathElement next; + + public SubfieldTokenizer(String path) + { + this.path = requireNonNull(path, "path is null"); + + if (path.isEmpty()) { + throw invalidSubfieldPath(); + } + } + + @Override + public final boolean hasNext() + { + if (state == State.FAILED) { + throw new IllegalStateException(); + } + switch (state) { + case DONE: + return false; + case READY: + return true; + default: + } + return tryToComputeNext(); + } + + private boolean tryToComputeNext() + { + state = State.FAILED; // temporary pessimism + next = computeNext(); + if (state != State.DONE) { + state = State.READY; + return true; + } + return false; + } + + @Override + public final Subfield.PathElement next() + { + if (!hasNext()) { + throw new NoSuchElementException(); + } + state = State.NOT_READY; + Subfield.PathElement result = next; + next = null; + return result; + } + + @Override + public final void remove() + { + throw new UnsupportedOperationException(); + } + + private Subfield.PathElement computeNext() + { + if (!hasNextCharacter()) { + state = State.DONE; + return null; + } + + if (tryMatch(DOT)) { + Subfield.PathElement token = matchPathSegment(); + firstSegment = false; + return token; + } + + if (tryMatch(OPEN_BRACKET)) { + Subfield.PathElement token = tryMatch(QUOTE) ? matchQuotedSubscript() : tryMatch(WILDCARD) ? matchWildcardSubscript() : matchUnquotedSubscript(); + + match(CLOSE_BRACKET); + firstSegment = false; + return token; + } + + if (firstSegment) { + Subfield.PathElement token = matchPathSegment(); + firstSegment = false; + return token; + } + + throw invalidSubfieldPath(); + } + + private Subfield.PathElement matchPathSegment() + { + // seek until we see a special character or whitespace + int start = index; + while (hasNextCharacter() && isUnquotedPathCharacter(peekCharacter())) { + nextCharacter(); + } + int end = index; + + String token = path.substring(start, end); + + // an empty unquoted token is not allowed + if (token.isEmpty()) { + throw invalidSubfieldPath(); + } + + return new Subfield.NestedField(token); + } + + private Subfield.PathElement matchWildcardSubscript() + { + return Subfield.allSubscripts(); + } + + private static boolean isUnquotedPathCharacter(char c) + { + return c == ':' || isUnquotedSubscriptCharacter(c); + } + + private Subfield.PathElement matchUnquotedSubscript() + { + // seek until we see a special character or whitespace + int start = index; + while (hasNextCharacter() && isUnquotedSubscriptCharacter(peekCharacter())) { + nextCharacter(); + } + int end = index; + + String token = path.substring(start, end); + + // an empty unquoted token is not allowed + if (token.isEmpty()) { + throw invalidSubfieldPath(); + } + + long index; + try { + index = Long.valueOf(token); + } + catch (NumberFormatException e) { + throw invalidSubfieldPath(); + } + + return new Subfield.LongSubscript(index); + } + + private static boolean isUnquotedSubscriptCharacter(char c) + { + return c == '_' || isLetterOrDigit(c); + } + + private Subfield.PathElement matchQuotedSubscript() + { + // quote has already been matched + + // seek until we see the close quote + StringBuilder token = new StringBuilder(); + boolean escaped = false; + + while (hasNextCharacter() && (escaped || peekCharacter() != QUOTE)) { + if (escaped) { + switch (peekCharacter()) { + case QUOTE: + case BACKSLASH: + token.append(peekCharacter()); + break; + default: + throw invalidSubfieldPath(); + } + escaped = false; + } + else { + if (peekCharacter() == BACKSLASH) { + escaped = true; + } + else { + token.append(peekCharacter()); + } + } + nextCharacter(); + } + if (escaped) { + throw invalidSubfieldPath(); + } + + match(QUOTE); + + String index = token.toString(); + if (index.equals(String.valueOf(WILDCARD))) { + return Subfield.allSubscripts(); + } + return new Subfield.StringSubscript(index); + } + + private boolean hasNextCharacter() + { + return index < path.length(); + } + + private void match(char expected) + { + if (!tryMatch(expected)) { + throw invalidSubfieldPath(); + } + } + + private boolean tryMatch(char expected) + { + if (!hasNextCharacter() || peekCharacter() != expected) { + return false; + } + index++; + return true; + } + + private void nextCharacter() + { + index++; + } + + private char peekCharacter() + { + return path.charAt(index); + } + + private PrestoException invalidSubfieldPath() + { + return new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid subfield path: '%s'", this)); + } + + @Override + public String toString() + { + return path.substring(0, index) + UNICODE_CARET + path.substring(index); + } + + private enum State { + /** We have computed the next element and haven't returned it yet. */ + READY, + + /** We haven't yet computed or have already returned the element. */ + NOT_READY, + + /** We have reached the end of the data and are finished. */ + DONE, + + /** We've suffered an exception and are kaput. */ + FAILED, + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java index 35446bd280f8a..26a1458d8b967 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java @@ -15,6 +15,7 @@ package com.facebook.presto.spi.block; import com.facebook.presto.spi.type.Type; +import org.openjdk.jol.info.ClassLayout; import javax.annotation.Nullable; @@ -28,6 +29,7 @@ import static com.facebook.presto.spi.block.BlockUtil.compactArray; import static com.facebook.presto.spi.block.BlockUtil.compactOffsets; import static com.facebook.presto.spi.block.MapBlock.createMapBlockInternal; +import static io.airlift.slice.SizeOf.sizeOfIntArray; import static java.util.Objects.requireNonNull; public abstract class AbstractMapBlock @@ -39,21 +41,23 @@ public abstract class AbstractMapBlock protected final Type keyType; protected final MethodHandle keyNativeHashCode; protected final MethodHandle keyBlockNativeEquals; + protected final MethodHandle keyBlockHashCode; - public AbstractMapBlock(Type keyType, MethodHandle keyNativeHashCode, MethodHandle keyBlockNativeEquals) + public AbstractMapBlock(Type keyType, MethodHandle keyNativeHashCode, MethodHandle keyBlockNativeEquals, MethodHandle keyBlockHashCode) { this.keyType = requireNonNull(keyType, "keyType is null"); // keyNativeHashCode can only be null due to map block kill switch. deprecated.new-map-block this.keyNativeHashCode = keyNativeHashCode; // keyBlockNativeEquals can only be null due to map block kill switch. deprecated.new-map-block this.keyBlockNativeEquals = keyBlockNativeEquals; + this.keyBlockHashCode = requireNonNull(keyBlockHashCode, "keyBlockHashCode is null"); } protected abstract Block getRawKeyBlock(); protected abstract Block getRawValueBlock(); - protected abstract int[] getHashTables(); + protected abstract HashTables getHashTables(); /** * offset is entry-based, not position-based. In other words, @@ -70,6 +74,8 @@ public AbstractMapBlock(Type keyType, MethodHandle keyNativeHashCode, MethodHand @Nullable protected abstract boolean[] getMapIsNull(); + protected abstract void ensureHashTableLoaded(); + int getOffset(int position) { return getOffsets()[position + getOffsetBase()]; @@ -111,22 +117,37 @@ public Block copyPositions(int[] positions, int offset, int length) newPosition++; } - int[] hashTable = getHashTables(); - int[] newHashTable = new int[newOffsets[newOffsets.length - 1] * HASH_MULTIPLIER]; - int newHashIndex = 0; - for (int i = offset; i < offset + length; ++i) { - int position = positions[i]; - int entriesStartOffset = getOffset(position); - int entriesEndOffset = getOffset(position + 1); - for (int hashIndex = entriesStartOffset * HASH_MULTIPLIER; hashIndex < entriesEndOffset * HASH_MULTIPLIER; hashIndex++) { - newHashTable[newHashIndex] = hashTable[hashIndex]; - newHashIndex++; + int[] rawHashTables = getHashTables().get().orElse(null); + int[] newRawHashTables = null; + int newHashTableEntries = newOffsets[newOffsets.length - 1] * HASH_MULTIPLIER; + if (rawHashTables != null) { + newRawHashTables = new int[newHashTableEntries]; + int newHashIndex = 0; + for (int i = offset; i < offset + length; ++i) { + int position = positions[i]; + int entriesStartOffset = getOffset(position); + int entriesEndOffset = getOffset(position + 1); + for (int hashIndex = entriesStartOffset * HASH_MULTIPLIER; hashIndex < entriesEndOffset * HASH_MULTIPLIER; hashIndex++) { + newRawHashTables[newHashIndex] = rawHashTables[hashIndex]; + newHashIndex++; + } } } Block newKeys = getRawKeyBlock().copyPositions(entriesPositions.elements(), 0, entriesPositions.size()); Block newValues = getRawValueBlock().copyPositions(entriesPositions.elements(), 0, entriesPositions.size()); - return createMapBlockInternal(0, length, Optional.of(newMapIsNull), newOffsets, newKeys, newValues, newHashTable, keyType, keyBlockNativeEquals, keyNativeHashCode); + return createMapBlockInternal( + 0, + length, + Optional.of(newMapIsNull), + newOffsets, + newKeys, + newValues, + new HashTables(Optional.ofNullable(newRawHashTables), newHashTableEntries), + keyType, + keyBlockNativeEquals, + keyNativeHashCode, + keyBlockHashCode); } @Override @@ -145,7 +166,8 @@ public Block getRegion(int position, int length) getHashTables(), keyType, keyBlockNativeEquals, - keyNativeHashCode); + keyNativeHashCode, + keyBlockHashCode); } @Override @@ -161,7 +183,8 @@ public long getRegionSizeInBytes(int position, int length) return getRawKeyBlock().getRegionSizeInBytes(entriesStart, entryCount) + getRawValueBlock().getRegionSizeInBytes(entriesStart, entryCount) + (Integer.BYTES + Byte.BYTES) * (long) length + - Integer.BYTES * HASH_MULTIPLIER * (long) entryCount; + Integer.BYTES * HASH_MULTIPLIER * (long) entryCount + + getHashTables().getInstanceSizeInBytes(); } @Override @@ -191,7 +214,8 @@ public long getPositionsSizeInBytes(boolean[] positions) return getRawKeyBlock().getPositionsSizeInBytes(entryPositions) + getRawValueBlock().getPositionsSizeInBytes(entryPositions) + (Integer.BYTES + Byte.BYTES) * (long) usedPositionCount + - Integer.BYTES * HASH_MULTIPLIER * (long) usedEntryCount; + Integer.BYTES * HASH_MULTIPLIER * (long) usedEntryCount + + getHashTables().getInstanceSizeInBytes(); } @Override public Block copyRegion(int position, int length) @@ -207,9 +231,15 @@ public Block copyRegion(int position, int length) int[] newOffsets = compactOffsets(getOffsets(), position + getOffsetBase(), length); boolean[] mapIsNull = getMapIsNull(); boolean[] newMapIsNull = mapIsNull == null ? null : compactArray(mapIsNull, position + getOffsetBase(), length); - int[] newHashTable = compactArray(getHashTables(), startValueOffset * HASH_MULTIPLIER, (endValueOffset - startValueOffset) * HASH_MULTIPLIER); - if (newKeys == getRawKeyBlock() && newValues == getRawValueBlock() && newOffsets == getOffsets() && newMapIsNull == mapIsNull && newHashTable == getHashTables()) { + int[] rawHashTables = getHashTables().get().orElse(null); + int[] newRawHashTables = null; + int expectedNewHashTableEntries = (endValueOffset - startValueOffset) * HASH_MULTIPLIER; + if (rawHashTables != null) { + newRawHashTables = compactArray(rawHashTables, startValueOffset * HASH_MULTIPLIER, expectedNewHashTableEntries); + } + + if (newKeys == getRawKeyBlock() && newValues == getRawValueBlock() && newOffsets == getOffsets() && newMapIsNull == mapIsNull && newRawHashTables == rawHashTables) { return this; } return createMapBlockInternal( @@ -219,10 +249,11 @@ public Block copyRegion(int position, int length) newOffsets, newKeys, newValues, - newHashTable, + new HashTables(Optional.ofNullable(newRawHashTables), expectedNewHashTableEntries), keyType, keyBlockNativeEquals, - keyNativeHashCode); + keyNativeHashCode, + keyBlockHashCode); } @Override @@ -238,12 +269,7 @@ public T getObject(int position, Class clazz) return clazz.cast(new SingleMapBlock( startEntryOffset * 2, (endEntryOffset - startEntryOffset) * 2, - getRawKeyBlock(), - getRawValueBlock(), - getHashTables(), - keyType, - keyNativeHashCode, - keyBlockNativeEquals)); + this)); } @Override @@ -263,7 +289,13 @@ public Block getSingleValueBlock(int position) int valueLength = endValueOffset - startValueOffset; Block newKeys = getRawKeyBlock().copyRegion(startValueOffset, valueLength); Block newValues = getRawValueBlock().copyRegion(startValueOffset, valueLength); - int[] newHashTable = Arrays.copyOfRange(getHashTables(), startValueOffset * HASH_MULTIPLIER, endValueOffset * HASH_MULTIPLIER); + + int[] rawHashTables = getHashTables().get().orElse(null); + int[] newRawHashTables = null; + int expectedNewHashTableEntries = (endValueOffset - startValueOffset) * HASH_MULTIPLIER; + if (rawHashTables != null) { + newRawHashTables = Arrays.copyOfRange(rawHashTables, startValueOffset * HASH_MULTIPLIER, endValueOffset * HASH_MULTIPLIER); + } return createMapBlockInternal( 0, @@ -272,10 +304,11 @@ public Block getSingleValueBlock(int position) new int[] {0, valueLength}, newKeys, newValues, - newHashTable, + new HashTables(Optional.ofNullable(newRawHashTables), expectedNewHashTableEntries), keyType, keyBlockNativeEquals, - keyNativeHashCode); + keyNativeHashCode, + keyBlockHashCode); } @Override @@ -314,10 +347,63 @@ public boolean isNull(int position) return mapIsNull != null && mapIsNull[position + getOffsetBase()]; } + public boolean isHashTablesPresent() + { + return getHashTables().get().isPresent(); + } + private void checkReadablePosition(int position) { if (position < 0 || position >= getPositionCount()) { throw new IllegalArgumentException("position is not valid"); } } + + public static class HashTables + { + private static final int INSTANCE_SIZE = ClassLayout.parseClass(HashTables.class).instanceSize(); + + // Hash to location in map. Writes to the field by MapBlock is protected by "HashTables" monitor in MapBlock. + // MapBlockBuilder instances have their dedicated hashTables instances, so the write accesses to the hashTables + // fields do not need to be synchronized in that class. + @Nullable + private volatile int[] hashTables; + + // The number of entries of hashTables array as if they are always built. It's used to calculate the retained size. + private int expectedEntryCount; + + HashTables(Optional hashTables, int expectedEntryCount) + { + if (hashTables.isPresent() && hashTables.get().length != expectedEntryCount) { + throw new IllegalArgumentException("hashTables size does not match expectedEntryCount"); + } + + this.hashTables = hashTables.orElse(null); + this.expectedEntryCount = expectedEntryCount; + } + + Optional get() + { + return Optional.ofNullable(hashTables); + } + + void set(int[] hashTables) + { + requireNonNull(hashTables, "hashTables is null"); + this.hashTables = hashTables; + + // The passed in hashTables are always sized as if they are fully built. + this.expectedEntryCount = hashTables.length; + } + + public long getInstanceSizeInBytes() + { + return INSTANCE_SIZE; + } + + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOfIntArray(expectedEntryCount); + } + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlock.java index 0a51354f1b46c..c2ab092e337ff 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlock.java @@ -42,7 +42,7 @@ public class MapBlock private final int[] offsets; private final Block keyBlock; private final Block valueBlock; - private final int[] hashTables; // hash to location in map; + private HashTables hashTables; private volatile long sizeInBytes; private final long retainedSizeInBytes; @@ -68,20 +68,6 @@ public static MapBlock fromKeyValueBlock( validateConstructorArguments(0, offsets.length - 1, mapIsNull.orElse(null), offsets, keyBlock, valueBlock, mapType.getKeyType(), keyBlockNativeEquals, keyNativeHashCode); int mapCount = offsets.length - 1; - int elementCount = keyBlock.getPositionCount(); - int[] hashTables = new int[elementCount * HASH_MULTIPLIER]; - Arrays.fill(hashTables, -1); - for (int i = 0; i < mapCount; i++) { - int keyOffset = offsets[i]; - int keyCount = offsets[i + 1] - keyOffset; - if (keyCount < 0) { - throw new IllegalArgumentException(format("Offset is not monotonically ascending. offsets[%s]=%s, offsets[%s]=%s", i, offsets[i], i + 1, offsets[i + 1])); - } - if (mapIsNull.isPresent() && mapIsNull.get()[i] && keyCount != 0) { - throw new IllegalArgumentException("A null map must have zero entries"); - } - buildHashTable(keyBlock, keyOffset, keyCount, keyBlockHashCode, hashTables, keyOffset * HASH_MULTIPLIER, keyCount * HASH_MULTIPLIER); - } return createMapBlockInternal( 0, @@ -90,10 +76,11 @@ public static MapBlock fromKeyValueBlock( offsets, keyBlock, valueBlock, - hashTables, + new HashTables(Optional.empty(), keyBlock.getPositionCount() * HASH_MULTIPLIER), mapType.getKeyType(), keyBlockNativeEquals, - keyNativeHashCode); + keyNativeHashCode, + keyBlockHashCode); } /** @@ -112,13 +99,26 @@ public static MapBlock createMapBlockInternal( int[] offsets, Block keyBlock, Block valueBlock, - int[] hashTables, + HashTables hashTables, Type keyType, MethodHandle keyBlockNativeEquals, - MethodHandle keyNativeHashCode) + MethodHandle keyNativeHashCode, + MethodHandle keyBlockHashCode) { validateConstructorArguments(startOffset, positionCount, mapIsNull.orElse(null), offsets, keyBlock, valueBlock, keyType, keyBlockNativeEquals, keyNativeHashCode); - return new MapBlock(startOffset, positionCount, mapIsNull.orElse(null), offsets, keyBlock, valueBlock, hashTables, keyType, keyBlockNativeEquals, keyNativeHashCode); + requireNonNull(hashTables, "hashTables is null"); + return new MapBlock( + startOffset, + positionCount, + mapIsNull.orElse(null), + offsets, + keyBlock, + valueBlock, + hashTables, + keyType, + keyBlockNativeEquals, + keyNativeHashCode, + keyBlockHashCode); } private static void validateConstructorArguments( @@ -171,16 +171,17 @@ private MapBlock( int[] offsets, Block keyBlock, Block valueBlock, - int[] hashTables, + HashTables hashTables, Type keyType, MethodHandle keyBlockNativeEquals, - MethodHandle keyNativeHashCode) + MethodHandle keyNativeHashCode, + MethodHandle keyBlockHashCode) { - super(keyType, keyNativeHashCode, keyBlockNativeEquals); + super(keyType, keyNativeHashCode, keyBlockNativeEquals, keyBlockHashCode); - requireNonNull(hashTables, "hashTables is null"); - if (hashTables.length < keyBlock.getPositionCount() * HASH_MULTIPLIER) { - throw new IllegalArgumentException(format("keyBlock/valueBlock size does not match hash table size: %s %s", keyBlock.getPositionCount(), hashTables.length)); + int[] rawHashTables = hashTables.get().orElse(null); + if (rawHashTables != null && rawHashTables.length < keyBlock.getPositionCount() * HASH_MULTIPLIER) { + throw new IllegalArgumentException(format("keyBlock/valueBlock size does not match hash table size: %s %s", keyBlock.getPositionCount(), rawHashTables.length)); } this.startOffset = startOffset; @@ -190,9 +191,17 @@ private MapBlock( this.keyBlock = keyBlock; this.valueBlock = valueBlock; this.hashTables = hashTables; - this.sizeInBytes = -1; - this.retainedSizeInBytes = INSTANCE_SIZE + keyBlock.getRetainedSizeInBytes() + valueBlock.getRetainedSizeInBytes() + sizeOf(offsets) + sizeOf(mapIsNull) + sizeOf(hashTables); + + // We will add the hashtable size to the retained size even if it's not built yet. This could be overestimating + // but is necessary to avoid reliability issues. Currently the memory counting framework only pull the retained + // size once for each operator so updating in the middle of the processing would not work. + this.retainedSizeInBytes = INSTANCE_SIZE + + keyBlock.getRetainedSizeInBytes() + + valueBlock.getRetainedSizeInBytes() + + sizeOf(offsets) + + sizeOf(mapIsNull) + + hashTables.getRetainedSizeInBytes(); } @Override @@ -208,7 +217,7 @@ protected Block getRawValueBlock() } @Override - protected int[] getHashTables() + protected HashTables getHashTables() { return hashTables; } @@ -255,7 +264,8 @@ private void calculateSize() sizeInBytes = keyBlock.getRegionSizeInBytes(entriesStart, entryCount) + valueBlock.getRegionSizeInBytes(entriesStart, entryCount) + (Integer.BYTES + Byte.BYTES) * (long) this.positionCount + - Integer.BYTES * HASH_MULTIPLIER * (long) entryCount; + Integer.BYTES * HASH_MULTIPLIER * (long) entryCount + + hashTables.getInstanceSizeInBytes(); } @Override @@ -271,7 +281,7 @@ public void retainedBytesForEachPart(BiConsumer consumer) consumer.accept(valueBlock, valueBlock.getRetainedSizeInBytes()); consumer.accept(offsets, sizeOf(offsets)); consumer.accept(mapIsNull, sizeOf(mapIsNull)); - consumer.accept(hashTables, sizeOf(hashTables)); + consumer.accept(hashTables, hashTables.getRetainedSizeInBytes()); consumer.accept(this, (long) INSTANCE_SIZE); } @@ -306,6 +316,44 @@ public Block getLoadedBlock() hashTables, keyType, keyBlockNativeEquals, - keyNativeHashCode); + keyNativeHashCode, + keyBlockHashCode); + } + + @Override + protected void ensureHashTableLoaded() + { + if (this.hashTables.get().isPresent()) { + return; + } + + // We need to synchronize access to the hashTables field as it may be shared by multiple MapBlock instances. + synchronized (hashTables) { + if (this.hashTables.get().isPresent()) { + return; + } + + int[] hashTables = new int[getRawKeyBlock().getPositionCount() * HASH_MULTIPLIER]; + Arrays.fill(hashTables, -1); + for (int i = 0; i < offsets.length - 1; i++) { + int keyOffset = offsets[i]; + int keyCount = offsets[i + 1] - keyOffset; + if (keyCount < 0) { + throw new IllegalArgumentException(format("Offset is not monotonically ascending. offsets[%s]=%s, offsets[%s]=%s", i, offsets[i], i + 1, offsets[i + 1])); + } + if (mapIsNull != null && mapIsNull[i] && keyCount != 0) { + throw new IllegalArgumentException("A null map must have zero entries"); + } + buildHashTable( + getRawKeyBlock(), + keyOffset, + keyCount, + keyBlockHashCode, + hashTables, + keyOffset * HASH_MULTIPLIER, + keyCount * HASH_MULTIPLIER); + } + this.hashTables.set(hashTables); + } } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockBuilder.java index d4b41cea4d6e9..62ea238b037ac 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockBuilder.java @@ -39,7 +39,6 @@ public class MapBlockBuilder private static final int INSTANCE_SIZE = ClassLayout.parseClass(MapBlockBuilder.class).instanceSize(); private final MethodHandle keyBlockEquals; - private final MethodHandle keyBlockHashCode; @Nullable private final BlockBuilderStatus blockBuilderStatus; @@ -49,7 +48,7 @@ public class MapBlockBuilder private boolean[] mapIsNull; private final BlockBuilder keyBlockBuilder; private final BlockBuilder valueBlockBuilder; - private int[] hashTables; + private HashTables hashTables; private boolean currentEntryOpened; @@ -88,12 +87,11 @@ private MapBlockBuilder( BlockBuilder valueBlockBuilder, int[] offsets, boolean[] mapIsNull, - int[] hashTables) + int[] rawHashTables) { - super(keyType, keyNativeHashCode, keyBlockNativeEquals); + super(keyType, keyNativeHashCode, keyBlockNativeEquals, keyBlockHashCode); this.keyBlockEquals = requireNonNull(keyBlockEquals, "keyBlockEquals is null"); - this.keyBlockHashCode = requireNonNull(keyBlockHashCode, "keyBlockHashCode is null"); this.blockBuilderStatus = blockBuilderStatus; this.positionCount = 0; @@ -101,7 +99,7 @@ private MapBlockBuilder( this.mapIsNull = requireNonNull(mapIsNull, "mapIsNull is null"); this.keyBlockBuilder = requireNonNull(keyBlockBuilder, "keyBlockBuilder is null"); this.valueBlockBuilder = requireNonNull(valueBlockBuilder, "valueBlockBuilder is null"); - this.hashTables = requireNonNull(hashTables, "hashTables is null"); + this.hashTables = new HashTables(Optional.of(requireNonNull(rawHashTables, "hashTables is null")), rawHashTables.length); } @Override @@ -117,7 +115,7 @@ protected Block getRawValueBlock() } @Override - protected int[] getHashTables() + protected HashTables getHashTables() { return hashTables; } @@ -151,13 +149,19 @@ public long getSizeInBytes() { return keyBlockBuilder.getSizeInBytes() + valueBlockBuilder.getSizeInBytes() + (Integer.BYTES + Byte.BYTES) * (long) positionCount + - Integer.BYTES * HASH_MULTIPLIER * (long) keyBlockBuilder.getPositionCount(); + Integer.BYTES * HASH_MULTIPLIER * (long) keyBlockBuilder.getPositionCount() + + hashTables.getInstanceSizeInBytes(); } @Override public long getRetainedSizeInBytes() { - long size = INSTANCE_SIZE + keyBlockBuilder.getRetainedSizeInBytes() + valueBlockBuilder.getRetainedSizeInBytes() + sizeOf(offsets) + sizeOf(mapIsNull) + sizeOf(hashTables); + long size = INSTANCE_SIZE + + keyBlockBuilder.getRetainedSizeInBytes() + + valueBlockBuilder.getRetainedSizeInBytes() + + sizeOf(offsets) + + sizeOf(mapIsNull) + + hashTables.getRetainedSizeInBytes(); if (blockBuilderStatus != null) { size += BlockBuilderStatus.INSTANCE_SIZE; } @@ -171,7 +175,7 @@ public void retainedBytesForEachPart(BiConsumer consumer) consumer.accept(valueBlockBuilder, valueBlockBuilder.getRetainedSizeInBytes()); consumer.accept(offsets, sizeOf(offsets)); consumer.accept(mapIsNull, sizeOf(mapIsNull)); - consumer.accept(hashTables, sizeOf(hashTables)); + consumer.accept(hashTables, hashTables.getRetainedSizeInBytes()); consumer.accept(this, (long) INSTANCE_SIZE); } @@ -199,7 +203,16 @@ public BlockBuilder closeEntry() int previousAggregatedEntryCount = offsets[positionCount - 1]; int aggregatedEntryCount = offsets[positionCount]; int entryCount = aggregatedEntryCount - previousAggregatedEntryCount; - buildHashTable(keyBlockBuilder, previousAggregatedEntryCount, entryCount, keyBlockHashCode, hashTables, previousAggregatedEntryCount * HASH_MULTIPLIER, entryCount * HASH_MULTIPLIER); + Optional rawHashTables = hashTables.get(); + verify(rawHashTables.isPresent(), "rawHashTables is empty"); + buildHashTable( + keyBlockBuilder, + previousAggregatedEntryCount, + entryCount, + keyBlockHashCode, + rawHashTables.get(), + previousAggregatedEntryCount * HASH_MULTIPLIER, + entryCount * HASH_MULTIPLIER); return this; } @@ -224,19 +237,21 @@ public BlockBuilder closeEntryStrict() int previousAggregatedEntryCount = offsets[positionCount - 1]; int aggregatedEntryCount = offsets[positionCount]; int entryCount = aggregatedEntryCount - previousAggregatedEntryCount; + Optional rawHashTables = hashTables.get(); + verify(rawHashTables.isPresent(), "rawHashTables is empty"); buildHashTableStrict( keyBlockBuilder, previousAggregatedEntryCount, entryCount, keyBlockEquals, keyBlockHashCode, - hashTables, + rawHashTables.get(), previousAggregatedEntryCount * HASH_MULTIPLIER, entryCount * HASH_MULTIPLIER); return this; } - private BlockBuilder closeEntry(int[] providedHashTable, int providedHashTableOffset) + private BlockBuilder closeEntry(@Nullable int[] providedHashTable, int providedHashTableOffset) { if (!currentEntryOpened) { throw new IllegalStateException("Expected entry to be opened but was closed"); @@ -246,14 +261,31 @@ private BlockBuilder closeEntry(int[] providedHashTable, int providedHashTableOf currentEntryOpened = false; ensureHashTableSize(); + int previousAggregatedEntryCount = offsets[positionCount - 1]; + int aggregatedEntryCount = offsets[positionCount]; - // Directly copy instead of building hashtable - int hashTableOffset = offsets[positionCount - 1] * HASH_MULTIPLIER; - int hashTableSize = (offsets[positionCount] - offsets[positionCount - 1]) * HASH_MULTIPLIER; - for (int i = 0; i < hashTableSize; i++) { - hashTables[hashTableOffset + i] = providedHashTable[providedHashTableOffset + i]; + Optional rawHashTables = hashTables.get(); + verify(rawHashTables.isPresent(), "rawHashTables is empty"); + if (providedHashTable != null) { + // Directly copy instead of building hashtable if providedHashTable is not null + int hashTableOffset = previousAggregatedEntryCount * HASH_MULTIPLIER; + int hashTableSize = (aggregatedEntryCount - previousAggregatedEntryCount) * HASH_MULTIPLIER; + for (int i = 0; i < hashTableSize; i++) { + rawHashTables.get()[hashTableOffset + i] = providedHashTable[providedHashTableOffset + i]; + } + } + else { + // Build hash table for this map entry. + int entryCount = aggregatedEntryCount - previousAggregatedEntryCount; + buildHashTable( + keyBlockBuilder, + previousAggregatedEntryCount, + entryCount, + keyBlockHashCode, + rawHashTables.get(), + previousAggregatedEntryCount * HASH_MULTIPLIER, + entryCount * HASH_MULTIPLIER); } - return this; } @@ -290,11 +322,13 @@ private void entryAdded(boolean isNull) private void ensureHashTableSize() { - if (hashTables.length < offsets[positionCount] * HASH_MULTIPLIER) { + Optional rawHashTables = hashTables.get(); + verify(rawHashTables.isPresent(), "rawHashTables is empty"); + if (rawHashTables.get().length < offsets[positionCount] * HASH_MULTIPLIER) { int newSize = BlockUtil.calculateNewArraySize(offsets[positionCount] * HASH_MULTIPLIER); - int oldSize = hashTables.length; - hashTables = Arrays.copyOf(hashTables, newSize); - Arrays.fill(hashTables, oldSize, hashTables.length, -1); + int[] newRawHashTables = Arrays.copyOf(rawHashTables.get(), newSize); + Arrays.fill(newRawHashTables, rawHashTables.get().length, newSize, -1); + hashTables.set(newRawHashTables); } } @@ -304,6 +338,10 @@ public Block build() if (currentEntryOpened) { throw new IllegalStateException("Current entry must be closed before the block can be built"); } + + Optional rawHashTables = hashTables.get(); + verify(rawHashTables.isPresent(), "rawHashTables is empty"); + int hashTablesEntries = offsets[positionCount] * HASH_MULTIPLIER; return createMapBlockInternal( 0, positionCount, @@ -311,9 +349,11 @@ public Block build() offsets, keyBlockBuilder.build(), valueBlockBuilder.build(), - Arrays.copyOf(hashTables, offsets[positionCount] * HASH_MULTIPLIER), + new HashTables(Optional.of(Arrays.copyOf(rawHashTables.get(), hashTablesEntries)), hashTablesEntries), keyType, - keyBlockNativeEquals, keyNativeHashCode); + keyBlockNativeEquals, + keyNativeHashCode, + keyBlockHashCode); } @Override @@ -388,7 +428,7 @@ public BlockBuilder appendStructureInternal(Block block, int position) } } - closeEntry(mapBlock.getHashTables(), startValueOffset * HASH_MULTIPLIER); + closeEntry(mapBlock.getHashTables().get().orElse(null), startValueOffset * HASH_MULTIPLIER); return this; } @@ -410,6 +450,9 @@ public BlockBuilder newBlockBuilderLike(BlockBuilderStatus blockBuilderStatus) newNegativeOneFilledArray(newSize * HASH_MULTIPLIER)); } + @Override + protected void ensureHashTableLoaded() {} + private static int[] newNegativeOneFilledArray(int size) { int[] hashTable = new int[size]; @@ -513,4 +556,11 @@ static int computePosition(long hashcode, int hashTableSize) { return (int) ((Integer.toUnsignedLong(Long.hashCode(hashcode)) * hashTableSize) >> 32); } + + private static void verify(boolean assertion, String message) + { + if (!assertion) { + throw new AssertionError(message); + } + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java index cc0b43537b653..f2db717216ff8 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java @@ -14,6 +14,7 @@ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.block.AbstractMapBlock.HashTables; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -54,7 +55,7 @@ public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceO int offsetBase = mapBlock.getOffsetBase(); int[] offsets = mapBlock.getOffsets(); - int[] hashTable = mapBlock.getHashTables(); + int[] hashTable = mapBlock.getHashTables().get().orElse(null); int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; @@ -64,8 +65,15 @@ public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceO blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawKeyBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawValueBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); - sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); - sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); + if (hashTable != null) { + int hashTableLength = (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER; + sliceOutput.appendInt(hashTableLength); // hashtable length + sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, hashTableLength)); + } + else { + // if the hashTable is null, we write the length -1 + sliceOutput.appendInt(-1); // hashtable length + } sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { @@ -82,18 +90,27 @@ public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceIn Block keyBlock = blockEncodingSerde.readBlock(sliceInput); Block valueBlock = blockEncodingSerde.readBlock(sliceInput); - int[] hashTable = new int[sliceInput.readInt()]; - sliceInput.readBytes(wrappedIntArray(hashTable)); + int hashTableLength = sliceInput.readInt(); + int[] hashTable = null; + if (hashTableLength >= 0) { + hashTable = new int[hashTableLength]; + sliceInput.readBytes(wrappedIntArray(hashTable)); + } + + if (keyBlock.getPositionCount() != valueBlock.getPositionCount()) { + throw new IllegalArgumentException( + format("Deserialized MapBlock violates invariants: key %d, value %d", keyBlock.getPositionCount(), valueBlock.getPositionCount())); + } - if (keyBlock.getPositionCount() != valueBlock.getPositionCount() || keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { + if (hashTable != null && keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { throw new IllegalArgumentException( - format("Deserialized MapBlock violates invariants: key %d, value %d, hash %d", keyBlock.getPositionCount(), valueBlock.getPositionCount(), hashTable.length)); + format("Deserialized MapBlock violates invariants: expected hashtable size %d, actual hashtable size %d", keyBlock.getPositionCount() * HASH_MULTIPLIER, hashTable.length)); } int positionCount = sliceInput.readInt(); int[] offsets = new int[positionCount + 1]; sliceInput.readBytes(wrappedIntArray(offsets)); Optional mapIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); - return MapType.createMapBlockInternal(typeManager, keyType, 0, positionCount, mapIsNull, offsets, keyBlock, valueBlock, hashTable); + return MapType.createMapBlockInternal(typeManager, keyType, 0, positionCount, mapIsNull, offsets, keyBlock, valueBlock, new HashTables(Optional.ofNullable(hashTable), hashTableLength)); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java index 6abae88e01329..1a6d91775eead 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlock.java @@ -19,14 +19,14 @@ import io.airlift.slice.Slice; import org.openjdk.jol.info.ClassLayout; -import java.lang.invoke.MethodHandle; +import javax.annotation.Nullable; + import java.util.function.BiConsumer; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.block.AbstractMapBlock.HASH_MULTIPLIER; import static com.facebook.presto.spi.block.MapBlockBuilder.computePosition; -import static io.airlift.slice.SizeOf.sizeOf; import static io.airlift.slice.SizeOf.sizeOfIntArray; import static java.lang.String.format; @@ -36,24 +36,14 @@ public class SingleMapBlock private static final int INSTANCE_SIZE = ClassLayout.parseClass(SingleMapBlock.class).instanceSize(); private final int offset; - private final int positionCount; - private final Block keyBlock; - private final Block valueBlock; - private final int[] hashTable; - final Type keyType; - private final MethodHandle keyNativeHashCode; - private final MethodHandle keyBlockNativeEquals; - - SingleMapBlock(int offset, int positionCount, Block keyBlock, Block valueBlock, int[] hashTable, Type keyType, MethodHandle keyNativeHashCode, MethodHandle keyBlockNativeEquals) + private final int positionCount; // The number of keys in this single map * 2 + private final AbstractMapBlock mapBlock; + + SingleMapBlock(int offset, int positionCount, AbstractMapBlock mapBlock) { this.offset = offset; this.positionCount = positionCount; - this.keyBlock = keyBlock; - this.valueBlock = valueBlock; - this.hashTable = hashTable; - this.keyType = keyType; - this.keyNativeHashCode = keyNativeHashCode; - this.keyBlockNativeEquals = keyBlockNativeEquals; + this.mapBlock = mapBlock; } @Override @@ -65,23 +55,26 @@ public int getPositionCount() @Override public long getSizeInBytes() { - return keyBlock.getRegionSizeInBytes(offset / 2, positionCount / 2) + - valueBlock.getRegionSizeInBytes(offset / 2, positionCount / 2) + + return mapBlock.getRawKeyBlock().getRegionSizeInBytes(offset / 2, positionCount / 2) + + mapBlock.getRawValueBlock().getRegionSizeInBytes(offset / 2, positionCount / 2) + sizeOfIntArray(positionCount / 2 * HASH_MULTIPLIER); } @Override public long getRetainedSizeInBytes() { - return INSTANCE_SIZE + keyBlock.getRetainedSizeInBytes() + valueBlock.getRetainedSizeInBytes() + sizeOf(hashTable); + return INSTANCE_SIZE + + mapBlock.getRawKeyBlock().getRetainedSizeInBytes() + + mapBlock.getRawValueBlock().getRetainedSizeInBytes() + + mapBlock.getHashTables().getRetainedSizeInBytes(); } @Override public void retainedBytesForEachPart(BiConsumer consumer) { - consumer.accept(keyBlock, keyBlock.getRetainedSizeInBytes()); - consumer.accept(valueBlock, valueBlock.getRetainedSizeInBytes()); - consumer.accept(hashTable, sizeOf(hashTable)); + consumer.accept(mapBlock.getRawKeyBlock(), mapBlock.getRawKeyBlock().getRetainedSizeInBytes()); + consumer.accept(mapBlock.getRawValueBlock(), mapBlock.getRawValueBlock().getRetainedSizeInBytes()); + consumer.accept(mapBlock.getHashTables(), mapBlock.getHashTables().getRetainedSizeInBytes()); consumer.accept(this, (long) INSTANCE_SIZE); } @@ -100,13 +93,13 @@ public int getOffset() @Override Block getRawKeyBlock() { - return keyBlock; + return mapBlock.getRawKeyBlock(); } @Override Block getRawValueBlock() { - return valueBlock; + return mapBlock.getRawValueBlock(); } @Override @@ -118,29 +111,30 @@ public String toString() @Override public Block getLoadedBlock() { - if (keyBlock != keyBlock.getLoadedBlock()) { + if (mapBlock.getRawKeyBlock() != mapBlock.getRawKeyBlock().getLoadedBlock()) { // keyBlock has to be loaded since MapBlock constructs hash table eagerly. throw new IllegalStateException(); } - Block loadedValueBlock = valueBlock.getLoadedBlock(); - if (loadedValueBlock == valueBlock) { + Block loadedValueBlock = mapBlock.getRawValueBlock().getLoadedBlock(); + if (loadedValueBlock == mapBlock.getRawValueBlock()) { return this; } return new SingleMapBlock( offset, positionCount, - keyBlock, - loadedValueBlock, - hashTable, - keyType, - keyNativeHashCode, - keyBlockNativeEquals); + mapBlock); } + @Nullable int[] getHashTable() { - return hashTable; + return mapBlock.getHashTables().get().orElse(null); + } + + Type getKeyType() + { + return mapBlock.keyType; } /** @@ -152,9 +146,12 @@ public int seekKey(Object nativeValue) return -1; } + mapBlock.ensureHashTableLoaded(); + int[] hashTable = mapBlock.getHashTables().get().get(); + long hashCode; try { - hashCode = (long) keyNativeHashCode.invoke(nativeValue); + hashCode = (long) mapBlock.keyNativeHashCode.invoke(nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -171,7 +168,7 @@ public int seekKey(Object nativeValue) Boolean match; try { // assuming maps with indeterminate keys are not supported - match = (Boolean) keyBlockNativeEquals.invoke(keyBlock, offset / 2 + keyPosition, nativeValue); + match = (Boolean) mapBlock.keyBlockNativeEquals.invoke(mapBlock.getRawKeyBlock(), offset / 2 + keyPosition, nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -196,9 +193,12 @@ public int seekKeyExact(long nativeValue) return -1; } + mapBlock.ensureHashTableLoaded(); + int[] hashTable = mapBlock.getHashTables().get().get(); + long hashCode; try { - hashCode = (long) keyNativeHashCode.invokeExact(nativeValue); + hashCode = (long) mapBlock.keyNativeHashCode.invokeExact(nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -215,7 +215,7 @@ public int seekKeyExact(long nativeValue) Boolean match; try { // assuming maps with indeterminate keys are not supported - match = (Boolean) keyBlockNativeEquals.invokeExact(keyBlock, offset / 2 + keyPosition, nativeValue); + match = (Boolean) mapBlock.keyBlockNativeEquals.invokeExact(mapBlock.getRawKeyBlock(), offset / 2 + keyPosition, nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -237,9 +237,12 @@ public int seekKeyExact(boolean nativeValue) return -1; } + mapBlock.ensureHashTableLoaded(); + int[] hashTable = mapBlock.getHashTables().get().get(); + long hashCode; try { - hashCode = (long) keyNativeHashCode.invokeExact(nativeValue); + hashCode = (long) mapBlock.keyNativeHashCode.invokeExact(nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -256,7 +259,7 @@ public int seekKeyExact(boolean nativeValue) Boolean match; try { // assuming maps with indeterminate keys are not supported - match = (Boolean) keyBlockNativeEquals.invokeExact(keyBlock, offset / 2 + keyPosition, nativeValue); + match = (Boolean) mapBlock.keyBlockNativeEquals.invokeExact(mapBlock.getRawKeyBlock(), offset / 2 + keyPosition, nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -278,9 +281,12 @@ public int seekKeyExact(double nativeValue) return -1; } + mapBlock.ensureHashTableLoaded(); + int[] hashTable = mapBlock.getHashTables().get().get(); + long hashCode; try { - hashCode = (long) keyNativeHashCode.invokeExact(nativeValue); + hashCode = (long) mapBlock.keyNativeHashCode.invokeExact(nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -297,7 +303,7 @@ public int seekKeyExact(double nativeValue) Boolean match; try { // assuming maps with indeterminate keys are not supported - match = (Boolean) keyBlockNativeEquals.invokeExact(keyBlock, offset / 2 + keyPosition, nativeValue); + match = (Boolean) mapBlock.keyBlockNativeEquals.invokeExact(mapBlock.getRawKeyBlock(), offset / 2 + keyPosition, nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -319,9 +325,12 @@ public int seekKeyExact(Slice nativeValue) return -1; } + mapBlock.ensureHashTableLoaded(); + int[] hashTable = mapBlock.getHashTables().get().get(); + long hashCode; try { - hashCode = (long) keyNativeHashCode.invokeExact(nativeValue); + hashCode = (long) mapBlock.keyNativeHashCode.invokeExact(nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -338,7 +347,7 @@ public int seekKeyExact(Slice nativeValue) Boolean match; try { // assuming maps with indeterminate keys are not supported - match = (Boolean) keyBlockNativeEquals.invokeExact(keyBlock, offset / 2 + keyPosition, nativeValue); + match = (Boolean) mapBlock.keyBlockNativeEquals.invokeExact(mapBlock.getRawKeyBlock(), offset / 2 + keyPosition, nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -360,9 +369,12 @@ public int seekKeyExact(Block nativeValue) return -1; } + mapBlock.ensureHashTableLoaded(); + int[] hashTable = mapBlock.getHashTables().get().get(); + long hashCode; try { - hashCode = (long) keyNativeHashCode.invokeExact(nativeValue); + hashCode = (long) mapBlock.keyNativeHashCode.invokeExact(nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); @@ -379,7 +391,7 @@ public int seekKeyExact(Block nativeValue) Boolean match; try { // assuming maps with indeterminate keys are not supported - match = (Boolean) keyBlockNativeEquals.invokeExact(keyBlock, offset / 2 + keyPosition, nativeValue); + match = (Boolean) mapBlock.keyBlockNativeEquals.invokeExact(mapBlock.getRawKeyBlock(), offset / 2 + keyPosition, nativeValue); } catch (Throwable throwable) { throw handleThrowable(throwable); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java index 0b28dd4b728f3..7407f17e65338 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java @@ -14,6 +14,7 @@ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.block.AbstractMapBlock.HashTables; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -22,6 +23,7 @@ import io.airlift.slice.SliceOutput; import java.lang.invoke.MethodHandle; +import java.util.Optional; import static com.facebook.presto.spi.block.AbstractMapBlock.HASH_MULTIPLIER; import static com.facebook.presto.spi.block.MethodHandleUtil.compose; @@ -54,15 +56,23 @@ public String getName() public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleMapBlock singleMapBlock = (SingleMapBlock) block; - TypeSerde.writeType(sliceOutput, singleMapBlock.keyType); + TypeSerde.writeType(sliceOutput, singleMapBlock.getKeyType()); int offset = singleMapBlock.getOffset(); int positionCount = singleMapBlock.getPositionCount(); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawKeyBlock().getRegion(offset / 2, positionCount / 2)); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawValueBlock().getRegion(offset / 2, positionCount / 2)); int[] hashTable = singleMapBlock.getHashTable(); - sliceOutput.appendInt(positionCount / 2 * HASH_MULTIPLIER); - sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, positionCount / 2 * HASH_MULTIPLIER)); + + if (hashTable != null) { + int hashTableLength = positionCount / 2 * HASH_MULTIPLIER; + sliceOutput.appendInt(hashTableLength); // hashtable length + sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, hashTableLength)); + } + else { + // if the hashTable is null, we write the length -1 + sliceOutput.appendInt(-1); + } } @Override @@ -72,18 +82,41 @@ public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceIn MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); + MethodHandle keyBlockHashCode = compose(keyNativeHashCode, nativeValueGetter(keyType)); Block keyBlock = blockEncodingSerde.readBlock(sliceInput); Block valueBlock = blockEncodingSerde.readBlock(sliceInput); - int[] hashTable = new int[sliceInput.readInt()]; - sliceInput.readBytes(wrappedIntArray(hashTable)); + int hashTableLength = sliceInput.readInt(); + int[] hashTable = null; + if (hashTableLength >= 0) { + hashTable = new int[hashTableLength]; + sliceInput.readBytes(wrappedIntArray(hashTable)); + } + + if (keyBlock.getPositionCount() != valueBlock.getPositionCount()) { + throw new IllegalArgumentException( + format("Deserialized SingleMapBlock violates invariants: key %d, value %d", keyBlock.getPositionCount(), valueBlock.getPositionCount())); + } - if (keyBlock.getPositionCount() != valueBlock.getPositionCount() || keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { + if (hashTable != null && keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { throw new IllegalArgumentException( - format("Deserialized SingleMapBlock violates invariants: key %d, value %d, hash %d", keyBlock.getPositionCount(), valueBlock.getPositionCount(), hashTable.length)); + format("Deserialized SingleMapBlock violates invariants: expected hashtable size %d, actual hashtable size %d", keyBlock.getPositionCount() * HASH_MULTIPLIER, hashTable.length)); } - return new SingleMapBlock(0, keyBlock.getPositionCount() * 2, keyBlock, valueBlock, hashTable, keyType, keyNativeHashCode, keyBlockNativeEquals); + MapBlock mapBlock = MapBlock.createMapBlockInternal( + 0, + 1, + Optional.empty(), + new int[] {0, keyBlock.getPositionCount()}, + keyBlock, + valueBlock, + new HashTables(Optional.ofNullable(hashTable), hashTableLength), + keyType, + keyBlockNativeEquals, + keyNativeHashCode, + keyBlockHashCode); + + return new SingleMapBlock(0, keyBlock.getPositionCount() * 2, mapBlock); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java index 886a9d88978fc..b213ac1096e04 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java @@ -16,6 +16,11 @@ import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.PredicateCompiler; +import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; public interface ConnectorContext @@ -30,6 +35,16 @@ default TypeManager getTypeManager() throw new UnsupportedOperationException(); } + default FunctionMetadataManager getFunctionMetadataManager() + { + throw new UnsupportedOperationException(); + } + + default StandardFunctionResolution getStandardFunctionResolution() + { + throw new UnsupportedOperationException(); + } + default PageSorter getPageSorter() { throw new UnsupportedOperationException(); @@ -39,4 +54,19 @@ default PageIndexerFactory getPageIndexerFactory() { throw new UnsupportedOperationException(); } + + default RowExpressionService getRowExpressionService() + { + throw new UnsupportedOperationException(); + } + + default DomainTranslator getDomainTranslator() + { + throw new UnsupportedOperationException(); + } + + default PredicateCompiler getPredicateCompiler() + { + throw new UnsupportedOperationException(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java index 6caf4b1a0aab3..806a90be50ab4 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java @@ -13,63 +13,14 @@ */ package com.facebook.presto.spi.function; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Objects; - -import static java.util.Objects.requireNonNull; +import com.facebook.presto.spi.CatalogSchemaName; /** * @apiNote FunctionHandle is a unique handle to identify the function implementation from namespaces. * However, currently it is still under changes, so please don't assume it is backward compatible. */ -public class FunctionHandle -{ - private final Signature signature; - - @JsonCreator - public FunctionHandle(@JsonProperty("signature") Signature signature) - { - this.signature = requireNonNull(signature, "signature is null"); - checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); - } - - @JsonProperty - public Signature getSignature() - { - return signature; - } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FunctionHandle that = (FunctionHandle) o; - return Objects.equals(signature, that.signature); - } - - @Override - public int hashCode() - { - return Objects.hash(signature); - } - - @Override - public String toString() - { - return signature.toString(); - } - - private static void checkArgument(boolean condition, String message, Object... args) - { - if (!condition) { - throw new IllegalArgumentException(String.format(message, args)); - } - } +public interface FunctionHandle +{ + CatalogSchemaName getCatalogSchemaName(); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java similarity index 91% rename from presto-main/src/main/java/com/facebook/presto/metadata/FunctionMetadata.java rename to presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java index b87642b8d3287..b4280fbd37c91 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java @@ -11,16 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.metadata; +package com.facebook.presto.spi.function; -import com.facebook.presto.spi.function.FunctionKind; -import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.TypeSignature; -import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import static java.util.Collections.unmodifiableList; import static java.util.Objects.requireNonNull; public class FunctionMetadata @@ -66,7 +65,7 @@ private FunctionMetadata( { this.name = requireNonNull(name, "name is null"); this.operatorType = requireNonNull(operatorType, "operatorType is null"); - this.argumentTypes = ImmutableList.copyOf(requireNonNull(argumentTypes, "argumentTypes is null")); + this.argumentTypes = unmodifiableList(new ArrayList<>(requireNonNull(argumentTypes, "argumentTypes is null"))); this.returnType = requireNonNull(returnType, "returnType is null"); this.functionKind = requireNonNull(functionKind, "functionKind is null"); this.deterministic = deterministic; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java new file mode 100644 index 0000000000000..04231b2c15557 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +public interface FunctionMetadataManager +{ + FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle); +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java new file mode 100644 index 0000000000000..ef6111daf2b94 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.type.Type; + +import java.util.List; + +public interface StandardFunctionResolution +{ + FunctionHandle notFunction(); + + boolean isNotFunction(FunctionHandle functionHandle); + + FunctionHandle negateFunction(Type type); + + boolean isNegateFunction(FunctionHandle functionHandle); + + FunctionHandle likeVarcharFunction(); + + FunctionHandle likeCharFunction(Type valueType); + + FunctionHandle arrayConstructor(List argumentTypes); + + FunctionHandle arithmeticFunction(OperatorType operator, Type leftType, Type rightType); + + boolean isArithmeticFunction(FunctionHandle functionHandle); + + FunctionHandle comparisonFunction(OperatorType operator, Type leftType, Type rightType); + + boolean isComparisonFunction(FunctionHandle functionHandle); + + FunctionHandle betweenFunction(Type valueType, Type lowerBoundType, Type upperBoundType); + + boolean isBetweenFunction(FunctionHandle functionHandle); + + FunctionHandle subscriptFunction(Type baseType, Type indexType); + + boolean isSubscriptFunction(FunctionHandle functionHandle); + + boolean isCastFunction(FunctionHandle functionHandle); +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/DefaultRowExpressionTraversalVisitor.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java similarity index 78% rename from presto-main/src/main/java/com/facebook/presto/sql/relational/DefaultRowExpressionTraversalVisitor.java rename to presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java index 945dc4071840a..5c4325498e621 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/DefaultRowExpressionTraversalVisitor.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java @@ -11,15 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.relational; - -import com.facebook.presto.spi.relation.CallExpression; -import com.facebook.presto.spi.relation.ConstantExpression; -import com.facebook.presto.spi.relation.InputReferenceExpression; -import com.facebook.presto.spi.relation.LambdaDefinitionExpression; -import com.facebook.presto.spi.relation.RowExpressionVisitor; -import com.facebook.presto.spi.relation.SpecialFormExpression; -import com.facebook.presto.spi.relation.VariableReferenceExpression; +package com.facebook.presto.spi.relation; /** * The default visitor serves as a template for "consumer-like" tree traversal. diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java new file mode 100644 index 0000000000000..af7e2491db778 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.relation; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.predicate.TupleDomain; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public interface DomainTranslator +{ + interface ColumnExtractor + { + Optional extract(RowExpression expression); + } + + RowExpression toPredicate(TupleDomain tupleDomain); + + /** + * Convert a RowExpression predicate into an ExtractionResult consisting of: + * 1) A successfully extracted TupleDomain + * 2) An RowExpression fragment which represents the part of the original RowExpression that will need to be re-evaluated + * after filtering with the TupleDomain. + */ + ExtractionResult fromPredicate(ConnectorSession session, RowExpression predicate, ColumnExtractor columnExtractor); + + class ExtractionResult + { + private final TupleDomain tupleDomain; + private final RowExpression remainingExpression; + + public ExtractionResult(TupleDomain tupleDomain, RowExpression remainingExpression) + { + this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); + this.remainingExpression = requireNonNull(remainingExpression, "remainingExpression is null"); + } + + public TupleDomain getTupleDomain() + { + return tupleDomain; + } + + public RowExpression getRemainingExpression() + { + return remainingExpression; + } + } + + ColumnExtractor BASIC_COLUMN_EXTRACTOR = expression -> { + if (expression instanceof VariableReferenceExpression) { + return Optional.of((VariableReferenceExpression) expression); + } + return Optional.empty(); + }; +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/Predicate.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/Predicate.java new file mode 100644 index 0000000000000..e96378ad0f2d3 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/Predicate.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.relation; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Page; + +public interface Predicate +{ + boolean isDeterministic(); + + int[] getInputChannels(); + + boolean evaluate(ConnectorSession session, Page page, int position); +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/PredicateCompiler.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/PredicateCompiler.java new file mode 100644 index 0000000000000..af36624554a9e --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/PredicateCompiler.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.relation; + +import java.util.Optional; +import java.util.function.Supplier; + +public interface PredicateCompiler +{ + /** + * Predicate expression may not contain any variable references, only input references. + */ + Supplier compilePredicate(RowExpression predicate, Optional classNameSuffix); +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java new file mode 100644 index 0000000000000..d39a2a944d969 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.relation; + +/** + * A set of services/utilities that are helpful for connectors to operate on row expressions + */ +public interface RowExpressionService +{ + DomainTranslator getDomainTranslator(); +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/MapType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/MapType.java index 8a6e4b940f687..1aa76106d7f05 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/MapType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/MapType.java @@ -14,6 +14,7 @@ package com.facebook.presto.spi.type; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.block.AbstractMapBlock.HashTables; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.BlockBuilderStatus; @@ -278,11 +279,11 @@ public static Block createMapBlockInternal( int[] offsets, Block keyBlock, Block valueBlock, - int[] hashTables) + HashTables hashTables) { // TypeManager caches types. Therefore, it is important that we go through it instead of coming up with the MethodHandles directly. // BIGINT is chosen arbitrarily here. Any type will do. MapType mapType = (MapType) typeManager.getType(new TypeSignature(StandardTypes.MAP, TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(BIGINT.getTypeSignature()))); - return MapBlock.createMapBlockInternal(startOffset, positionCount, mapIsNull, offsets, keyBlock, valueBlock, hashTables, keyType, mapType.keyBlockNativeEquals, mapType.keyNativeHashCode); + return MapBlock.createMapBlockInternal(startOffset, positionCount, mapIsNull, offsets, keyBlock, valueBlock, hashTables, keyType, mapType.keyBlockNativeEquals, mapType.keyNativeHashCode, mapType.keyBlockHashCode); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java index 498063f219fe4..22055f46f9e81 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java @@ -63,6 +63,15 @@ public static RowType anonymous(List types) return new RowType(makeSignature(fields), fields); } + public static RowType withDefaultFieldNames(List types) + { + List fields = new ArrayList<>(); + for (int i = 0; i < types.size(); i++) { + fields.add(new Field(Optional.of("field" + i), types.get(i))); + } + return new RowType(makeSignature(fields), fields); + } + // Only RowParametricType.createType should call this method public static RowType createWithTypeSignature(TypeSignature typeSignature, List fields) { diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/TestSubfieldTokenizer.java b/presto-spi/src/test/java/com/facebook/presto/spi/TestSubfieldTokenizer.java new file mode 100644 index 0000000000000..83718229a6f12 --- /dev/null +++ b/presto-spi/src/test/java/com/facebook/presto/spi/TestSubfieldTokenizer.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi; + +import com.facebook.presto.spi.Subfield.NestedField; +import com.facebook.presto.spi.Subfield.PathElement; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Streams; +import org.testng.annotations.Test; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class TestSubfieldTokenizer +{ + @Test + public void test() + { + List elements = ImmutableList.of( + new NestedField("b"), + new Subfield.LongSubscript(2), + new Subfield.StringSubscript("z"), + Subfield.allSubscripts(), + new Subfield.StringSubscript("34"), + new Subfield.StringSubscript("b \"test\""), + new Subfield.StringSubscript("\"abc"), + new Subfield.StringSubscript("abc\""), + new Subfield.StringSubscript("ab\"cde"), + new Subfield.StringSubscript("a.b[\"hello\uDBFF\"]")); + + for (PathElement element : elements) { + assertPath(new Subfield("a", ImmutableList.of(element))); + } + + for (PathElement element : elements) { + for (PathElement secondElement : elements) { + assertPath(new Subfield("a", ImmutableList.of(element, secondElement))); + } + } + + for (PathElement element : elements) { + for (PathElement secondElement : elements) { + for (PathElement thirdElement : elements) { + assertPath(new Subfield("a", ImmutableList.of(element, secondElement, thirdElement))); + } + } + } + } + + private static void assertPath(Subfield path) + { + SubfieldTokenizer tokenizer = new SubfieldTokenizer(path.serialize()); + assertTrue(tokenizer.hasNext()); + assertEquals(new Subfield(((NestedField) tokenizer.next()).getName(), Streams.stream(tokenizer).collect(toImmutableList())), path); + } + + @Test + public void testInvalidPaths() + { + assertInvalidPath("a[b]"); + assertInvalidPath("a[2"); + assertInvalidPath("a.*"); + assertInvalidPath("a[2].[3]."); + } + + private void assertInvalidPath(String path) + { + SubfieldTokenizer tokenizer = new SubfieldTokenizer(path); + + try { + Streams.stream(tokenizer).collect(toImmutableList()); + fail("Expected failure"); + } + catch (PrestoException e) { + // this is expected + assertTrue(e.getMessage().startsWith("Invalid subfield path: ")); + } + } +} diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java index e55cc1df6b89e..d2172c1924927 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java @@ -1269,4 +1269,60 @@ public void testOrderedAggregations() "('5-LOW', 445 , NULL)," + "('1-URGENT', 781 , ('O'))"); } + + /** + * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction + */ + @Test + public void testQuantileDigest() + { + assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(quantity), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL)), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey, 2), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(quantity, 3), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey, 2, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(quantity, 3, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); + } + + /** + * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction + */ + @Test + public void testQuantileDigestGroupBy() + { + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey, 2), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity, 3), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey, 2, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity, 3, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + } + + /** + * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction + */ + @Test + public void testQuantileDigestMerge() + { + assertQuery("SELECT value_at_quantile(merge(qdigest), 0.5E0) > 0 FROM (SELECT partkey, qdigest_agg(orderkey) as qdigest FROM lineitem GROUP BY partkey)", "SELECT true"); + } + + /** + * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction + */ + @Test + public void testQuantileDigestMergeGroupBy() + { + assertQuery("" + + "SELECT partkey, value_at_quantile(merge(qdigest), 0.5E0) > 0 " + + "FROM (SELECT partkey, suppkey, qdigest_agg(orderkey) as qdigest FROM lineitem GROUP BY partkey, suppkey)" + + "GROUP BY partkey", + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index cf6e8b3e70d3e..16ca2278a4d55 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -141,6 +141,12 @@ protected void assertQuery(Session session, @Language("SQL") String actual, @Lan QueryAssertions.assertQuery(queryRunner, session, actual, h2QueryRunner, expected, false, false); } + protected void assertQuery(Session session, @Language("SQL") String sql, Consumer planAssertion) + { + checkArgument(queryRunner instanceof DistributedQueryRunner, "pattern assertion is only supported for DistributedQueryRunner"); + QueryAssertions.assertQuery(queryRunner, session, sql, h2QueryRunner, sql, false, false, planAssertion); + } + protected void assertQuery(Session session, @Language("SQL") String actual, @Language("SQL") String expected, Consumer planAssertion) { checkArgument(queryRunner instanceof DistributedQueryRunner, "pattern assertion is only supported for DistributedQueryRunner"); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index dc80176ae95a6..fff505dd730df 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -22,12 +22,12 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.AllNodes; import com.facebook.presto.metadata.Catalog; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.server.testing.TestingPrestoServer; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.QueryId; import com.facebook.presto.split.PageSourceManager; @@ -332,7 +332,7 @@ private boolean isConnectionVisibleToAllNodes(ConnectorId connectorId) { for (TestingPrestoServer server : servers) { server.refreshNodes(); - Set activeNodesWithConnector = server.getActiveNodesWithConnector(connectorId); + Set activeNodesWithConnector = server.getActiveNodesWithConnector(connectorId); if (activeNodesWithConnector.size() != servers.size()) { return false; } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/H2QueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/H2QueryRunner.java index 0af1ba425b568..d3416d507f7c8 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/H2QueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/H2QueryRunner.java @@ -81,6 +81,7 @@ import static io.airlift.tpch.TpchTable.NATION; import static io.airlift.tpch.TpchTable.ORDERS; import static io.airlift.tpch.TpchTable.PART; +import static io.airlift.tpch.TpchTable.PART_SUPPLIER; import static io.airlift.tpch.TpchTable.REGION; import static java.lang.String.format; import static java.util.Collections.nCopies; @@ -130,6 +131,16 @@ public H2QueryRunner() ")"); insertRows(tpchMetadata, LINE_ITEM); + handle.execute(" CREATE TABLE partsupp (\n" + + " partkey BIGINT NOT NULL,\n" + + " suppkey BIGINT NOT NULL,\n" + + " availqty INTEGER NOT NULL,\n" + + " supplycost DOUBLE NOT NULL,\n" + + " comment VARCHAR(199) NOT NULL,\n" + + " PRIMARY KEY(partkey, suppkey)" + + ")"); + insertRows(tpchMetadata, PART_SUPPLIER); + handle.execute("CREATE TABLE nation (\n" + " nationkey BIGINT PRIMARY KEY,\n" + " name VARCHAR(25) NOT NULL,\n" + diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java index efea4c2cadb01..862d9f8c91192 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java @@ -17,11 +17,11 @@ import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.cost.StatsCalculator; import com.facebook.presto.metadata.AllNodes; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.server.testing.TestingPrestoServer; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.Plugin; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; @@ -190,7 +190,7 @@ public void refreshNodes() private void refreshNodes(ConnectorId connectorId) { - Set activeNodesWithConnector; + Set activeNodesWithConnector; do { try { diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index e49117926e4db..9ac5f048fe2f0 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -17,6 +17,11 @@ + + com.facebook.presto + presto-base-jdbc + + com.facebook.presto presto-hive diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java index da896c32f72e1..988507a94a19f 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.HiveErrorCode; import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.plugin.jdbc.JdbcErrorCode; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.StandardErrorCode; import com.google.common.collect.ImmutableSet; @@ -37,6 +38,7 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR; +import static com.facebook.presto.plugin.jdbc.JdbcErrorCode.JDBC_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; import static com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_ERROR; import static com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_TIMEOUT; @@ -55,6 +57,7 @@ public class PrestoExceptionClassifier private static final Set DEFAULT_ERRORS = ImmutableSet.builder() .addAll(asList(StandardErrorCode.values())) .addAll(asList(HiveErrorCode.values())) + .addAll(asList(JdbcErrorCode.values())) .build(); private static final Set DEFAULT_RETRYABLE_ERRORS = ImmutableSet.of( @@ -75,7 +78,9 @@ public class PrestoExceptionClassifier HIVE_WRITER_CLOSE_ERROR, HIVE_FILESYSTEM_ERROR, HIVE_CANNOT_OPEN_SPLIT, - HIVE_METASTORE_ERROR); + HIVE_METASTORE_ERROR, + // From JdbcErrorCode + JDBC_ERROR); private final Map errorByCode; private final Set retryableErrors; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java index b93c87ca8189b..ee9c0a2cbf896 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java @@ -33,8 +33,7 @@ public ExceededTimeLimitFailureResolver() @Override public Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats) { - if (errorCode == EXCEEDED_TIME_LIMIT && - controlQueryStats.getCpuTimeMillis() > testQueryStats.getCpuTimeMillis()) { + if (errorCode == EXCEEDED_TIME_LIMIT) { return Optional.of("Auto Resolved: Test cluster has less computing resource"); } return Optional.empty(); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java index a7449b9cfbfcc..e16c710af0edd 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java @@ -22,7 +22,6 @@ import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; public class TestExceededTimeLimitFailureResolver extends AbstractTestPrestoQueryFailureResolver @@ -32,20 +31,6 @@ public TestExceededTimeLimitFailureResolver() super(new ExceededTimeLimitFailureResolver()); } - @Test - public void testHigherCpuTime() - { - assertFalse(getFailureResolver().resolve( - CONTROL_QUERY_STATS, - QueryException.forPresto( - new RuntimeException(), - Optional.of(EXCEEDED_TIME_LIMIT), - false, - Optional.of(createQueryStats(CONTROL_CPU_TIME_MILLIS * 2, CONTROL_PEAK_MEMORY_BYTES)), - forMain(TEST))) - .isPresent()); - } - @Test public void testResolved() {